vendor/tramline

changeset 10:8255ba231de3 gracinet-progress-bar

Switch to stat info kept on FS Indeed mod_python RAM belongs to the server process.
author gracinet
date Sun, 22 Feb 2009 20:20:11 +0000
parents 0844398c9e18
children 0075e64fe07e
files src/tramline/core.py src/tramline/tests/test_core.py
diffstat 2 files changed, 39 insertions(+), 19 deletions(-) [+]
line diff
     1.1 --- a/src/tramline/core.py
     1.2 +++ b/src/tramline/core.py
     1.3 @@ -30,6 +30,9 @@
     1.4  def tramline_repository_path(req):
     1.5      return os.path.join(tramline_path(req), 'repository')
     1.6  
     1.7 +def tramline_stat_path(req):
     1.8 +    return os.path.join(tramline_path(req), 'stat')
     1.9 +
    1.10  def group_write(req): 
    1.11      op = req.get_options().get(OPTION_ALLOW_GROUP_WRITE) 
    1.12      return sys.platform != 'win32' \
    1.13 @@ -39,6 +42,7 @@
    1.14      base = tramline_path(req)
    1.15      for i, p in enumerate((tramline_path(req), 
    1.16                             tramline_upload_path(req), 
    1.17 +                           tramline_stat_path(req), 
    1.18                             tramline_repository_path(req))):
    1.19          if not os.path.isdir(p):
    1.20              os.mkdir(p)
    1.21 @@ -48,7 +52,7 @@
    1.22                  os.chmod(p, 02775)
    1.23  
    1.24  FILE_CHUNKSIZE = 8 * 1024
    1.25 -
    1.26 +REPORT_BLOCKSIZE = 100 * 1024
    1.27  """
    1.28  inputfilter() and outputfilter() are what is called by Apache.
    1.29  
    1.30 @@ -307,7 +311,6 @@
    1.31  class ProcessorRegistry:
    1.32      def __init__(self):
    1.33          self._processors = {}
    1.34 -        self._processors_by_progress_id = {}
    1.35  
    1.36      def getProcessor(self, id):
    1.37          return self._processors[id]
    1.38 @@ -321,22 +324,13 @@
    1.39          result = self._processors[id] = Processor(id)
    1.40          return result
    1.41      
    1.42 -    def registerProgressId(self, processor):
    1.43 -        self._processors_by_progress_id[processor.progress_id] = processor
    1.44 -        
    1.45      def removeProcessor(self, processor):
    1.46 -        prog_id = processor.progress_id
    1.47 -        if prog_id is not None:
    1.48 -            del self._processors_by_progress_id[prog_id]
    1.49          del self._processors[processor.id]
    1.50  
    1.51 -    def getUploaded(self, progress_id):
    1.52 -        proc = self._processors_by_progress_id.get(progress_id)
    1.53 -        if proc is None:
    1.54 -            return
    1.55 -        return proc.uploaded
    1.56 +theProcessorRegistry = ProcessorRegistry()
    1.57  
    1.58 -theProcessorRegistry = ProcessorRegistry()
    1.59 +def get_progress_path(req, progress_id):
    1.60 +    return os.path.join(tramline_stat_path(req), progress_id)
    1.61  
    1.62  class Processor:
    1.63      """Processor for Post requests. 
    1.64 @@ -352,15 +346,15 @@
    1.65             means to communicate it back once the process has started. 
    1.66             Still some effort is made to avoid malicious garbling of ids.
    1.67  
    1.68 -       upload: this is the total file data upload this processor has seen. 
    1.69 +       uploaded: this is the total file data upload this processor has seen. 
    1.70             there might be more than one file in this request.
    1.71      """
    1.72         
    1.73 -
    1.74      def __init__(self, id):
    1.75          self.uploaded = 0L
    1.76          self.id = id
    1.77          self.progress_id = None
    1.78 +        self._uploaded_blocks = 0 
    1.79          self._upload_files = []
    1.80          self._incoming = []
    1.81  	self._isize = 0
    1.82 @@ -375,6 +369,22 @@
    1.83          for line in lines:
    1.84              self.pushInputLine(line, out)
    1.85  
    1.86 +    def logProgress(self, req):
    1.87 +        """Log progress upload to a file for async requests to use
    1.88 +
    1.89 +        Report only every REPORT_BLOCKSIZE bytes to avoid too much I/O
    1.90 +        the file is located in the stat directory, and its name is progress_id.
    1.91 +        Race condition is in theory possible, but very unlikely, and not
    1.92 +        critical to avoid.
    1.93 +        """
    1.94 +        blocks = self.uploaded / REPORT_BLOCKSIZE
    1.95 +        if blocks > self._uploaded_blocks:
    1.96 +            self._uploaded_blocks = blocks
    1.97 +            if self.progress_id is not None:
    1.98 +                f = open(get_progress_path(req, self.progress_id), 'w')
    1.99 +                f.write(str(self.uploaded))
   1.100 +                f.close()
   1.101 +
   1.102      def pushInputLine(self, data, out):
   1.103          # collect data
   1.104          self._incoming.append(data)
   1.105 @@ -396,6 +406,7 @@
   1.106          self._isize = 0
   1.107  
   1.108          self.handle(line, out)
   1.109 +        self.logProgress(out.req)
   1.110  
   1.111      def finalizeInput(self, out):
   1.112          if self._upload_files:
   1.113 @@ -495,7 +506,6 @@
   1.114              # for stat requests, and not sure the user agent can know and store
   1.115              # the full socket info of the upload for requesting.
   1.116              self.progress_id = out.req.connection.remote_ip + '-' + line.strip()
   1.117 -            theProcessorRegistry.registerProgressId(self)
   1.118  
   1.119      def handle_data(self, line, out):
   1.120          out.write(line)
   1.121 @@ -529,6 +539,12 @@
   1.122                  self.uploaded += len(self._previous_line)
   1.123              self._previous_line = line
   1.124  
   1.125 +def get_progress(req, progress_id):
   1.126 +    f = open(get_progress_path(req, progress_id), 'r')
   1.127 +    s = f.read()
   1.128 +    f.close() # let's be explicit :-)
   1.129 +    return s
   1.130 +
   1.131  def parse_header(s):
   1.132      l = [e.strip() for e in s.split(';')]
   1.133      result_value = l.pop(0).lower()
     2.1 --- a/src/tramline/tests/test_core.py
     2.2 +++ b/src/tramline/tests/test_core.py
     2.3 @@ -11,6 +11,9 @@
     2.4  
     2.5  from tramline.core import theProcessorRegistry
     2.6  
     2.7 +from tramline import core as tramcore
     2.8 +tramcore.REPORT_BLOCKSIZE = 5
     2.9 +
    2.10  tramline_path = '/tmp/trampath'
    2.11  
    2.12  class StringTable(dict):
    2.13 @@ -152,7 +155,8 @@
    2.14  
    2.15          input.close()
    2.16  
    2.17 -        uploaded = theProcessorRegistry.getUploaded(progress_id)
    2.18 +        from tramline.core import get_progress
    2.19 +        uploaded = get_progress(filter.req, progress_id)
    2.20          self.assertFalse(uploaded is None)
    2.21  
    2.22          output_data = output.getvalue()
    2.23 @@ -165,7 +169,7 @@
    2.24          expected = 'first line\nsecond line\n'
    2.25          self.assertEquals('first line\nsecond line\n', data)
    2.26          # process has finished, so uploaded should be the entire length
    2.27 -        self.assertEquals(len(expected), uploaded)
    2.28 +        self.assertEquals(str(len(expected)), uploaded)
    2.29          
    2.30      def test_split_filter(self):
    2.31          f = open(get_data_path('input2.txt'), 'rb')