vendor/tramline

changeset 51:8f4cf193078b

General merge from 3.5 branch
author Georges Racinet on purity.racinet.fr <georges@racinet.fr>
date Sat, 29 Sep 2012 11:42:26 +0200
parents 47723198ebca f1129ec07877
children
files VERSION
diffstat 9 files changed, 379 insertions(+), 48 deletions(-) [+]
line diff
     1.1 new file mode 100644
     1.2 --- /dev/null
     1.3 +++ b/.hgignore
     1.4 @@ -0,0 +1,8 @@
     1.5 +syntax:glob
     1.6 +# usual stuff
     1.7 +*\#
     1.8 +*.\#*
     1.9 +*~
    1.10 +*.pyo
    1.11 +*.orig
    1.12 +*.pyc
     2.1 new file mode 100644
     2.2 --- /dev/null
     2.3 +++ b/.hgtags
     2.4 @@ -0,0 +1,5 @@
     2.5 +906c932e771a6aedf2e08b7bbaee576bff7c09fc 0.10.0-gracinet-fix-range
     2.6 +6fb6ef28e5fc47c0e55dbf734f36fcd7fbd3b9a8 0.11.0-gracinet-fix-range
     2.7 +34a1f6e461279bb2ef3aa061048ce6ed74120bd5 0.11.1-gracinet-fix-range
     2.8 +9cbd02795d8bfc6ac7251e38b82e288f2ad08fab 0.12.0-gracinet-fix-range
     2.9 +edc91981a82d3d4c53504775055576dcc0946fbe 0.13.0-CPS-3.5
     3.1 --- a/CHANGES
     3.2 +++ b/CHANGES
     3.3 @@ -3,11 +3,10 @@
     3.4  -
     3.5  New features
     3.6  ~~~~~~~~~~~~
     3.7 -- Moved filesystem path utils to a separate package for easy reference by
     3.8 -  svn externals
     3.9 +-
    3.10  Bug fixes
    3.11  ~~~~~~~~~
    3.12 -- #2017: download hangs with some browser plugins (PDF with Firefox for Windows)
    3.13 +-
    3.14  New internal features
    3.15  ~~~~~~~~~~~~~~~~~~~~~
    3.16  - 
     4.1 --- a/HISTORY
     4.2 +++ b/HISTORY
     4.3 @@ -1,3 +1,89 @@
     4.4 +===========================================================
     4.5 +Package: tramline-viral 0.13.0-CPS-3.5
     4.6 +===========================================================
     4.7 +First release built by: gracinet at: 2012-02-18T23:00:06
     4.8 +Requires
     4.9 +~~~~~~~~
    4.10 +-
    4.11 +New features
    4.12 +~~~~~~~~~~~~
    4.13 +- Creation of the stable CPS-3.5 branch
    4.14 +Bug fixes
    4.15 +~~~~~~~~~
    4.16 +-
    4.17 +New internal features
    4.18 +~~~~~~~~~~~~~~~~~~~~~
    4.19 +- 
    4.20 +
    4.21 +===========================================================
    4.22 +Package: tramline-viral 0.12.0-gracinet-fix-range
    4.23 +===========================================================
    4.24 +First release built by: gracinet at: 2011-02-01T21:24:21
    4.25 +Requires
    4.26 +~~~~~~~~
    4.27 +-
    4.28 +New features
    4.29 +~~~~~~~~~~~~
    4.30 +- #2322: PUT requests and support for ZopeExternalEditor
    4.31 +Bug fixes
    4.32 +~~~~~~~~~
    4.33 +-
    4.34 +New internal features
    4.35 +~~~~~~~~~~~~~~~~~~~~~
    4.36 +- 
    4.37 +
    4.38 +===========================================================
    4.39 +Package: tramline-viral 0.11.1-gracinet-fix-range
    4.40 +===========================================================
    4.41 +First release built by: gracinet at: 2010-10-13T11:38:55
    4.42 +Requires
    4.43 +~~~~~~~~
    4.44 +-
    4.45 +New features
    4.46 +~~~~~~~~~~~~
    4.47 +-
    4.48 +Bug fixes
    4.49 +~~~~~~~~~
    4.50 +- Logging now reasonible
    4.51 +New internal features
    4.52 +~~~~~~~~~~~~~~~~~~~~~
    4.53 +- 
    4.54 +
    4.55 +===========================================================
    4.56 +Package: tramline-viral 0.11.0-gracinet-fix-range
    4.57 +===========================================================
    4.58 +First release built by: gracinet at: 2010-08-24T18:30:12
    4.59 +Requires
    4.60 +~~~~~~~~
    4.61 +-
    4.62 +New features
    4.63 +~~~~~~~~~~~~
    4.64 +- #2222: support for upload stats (for progress bars)
    4.65 +Bug fixes
    4.66 +~~~~~~~~~
    4.67 +- 
    4.68 +New internal features
    4.69 +~~~~~~~~~~~~~~~~~~~~~
    4.70 +- 
    4.71 +
    4.72 +===========================================================
    4.73 +Package: tramline-viral 0.10.0-gracinet-fix-range
    4.74 +===========================================================
    4.75 +First release built by: gracinet at: 2010-03-15T08:32:04
    4.76 +Requires
    4.77 +~~~~~~~~
    4.78 +-
    4.79 +New features
    4.80 +~~~~~~~~~~~~
    4.81 +- Moved filesystem path utils to a separate package for easy reference by
    4.82 +  svn externals
    4.83 +Bug fixes
    4.84 +~~~~~~~~~
    4.85 +- #2017: download hangs with some browser plugins (PDF with Firefox for Windows)
    4.86 +New internal features
    4.87 +~~~~~~~~~~~~~~~~~~~~~
    4.88 +- 
    4.89 +
    4.90  ===========================================================
    4.91  Package: tramline-viral 0.9.0
    4.92  ===========================================================
     5.1 --- a/INSTALL.txt
     5.2 +++ b/INSTALL.txt
     5.3 @@ -11,9 +11,17 @@
     5.4    into Apache into your underlying appserver. It also works with
     5.5    mod_rewrite if you use proxying mode for this ([P]).
     5.6  
     5.7 -* mod_python, with the apache.py file patched so filter is not
     5.8 +* mod_python, MAYBE with the apache.py file patched so filter is not
     5.9    flushed. In mod_python/lib/python/apache.py, comment out
    5.10    'filter.flush()' in FilterDispatch.
    5.11 +  
    5.12 +  known GNU/Linux distributions where the patch is not needed and turns harmful
    5.13 +        - Debian lenny
    5.14 +
    5.15 +  known distributions where the patch is needed:
    5.16 +        - Debian etch
    5.17 +
    5.18 +  TODO: more reports
    5.19  
    5.20  Apache conf
    5.21  -----------
     6.1 --- a/src/tramline/core.py
     6.2 +++ b/src/tramline/core.py
     6.3 @@ -1,4 +1,6 @@
     6.4  import os, tempfile, random, sys, errno, mimetools
     6.5 +import cgi
     6.6 +from mod_python import apache, Cookie
     6.7  
     6.8  OPTION_ALLOW_GROUP_WRITE = 'allow_group_write'
     6.9  TRAMLINE_RANGE_HEADER = 'X-Tramline-Original-Range'
    6.10 @@ -30,6 +32,9 @@
    6.11  def tramline_repository_path(req):
    6.12      return os.path.join(tramline_path(req), 'repository')
    6.13  
    6.14 +def tramline_stat_path(req):
    6.15 +    return os.path.join(tramline_path(req), 'stat')
    6.16 +
    6.17  def group_write(req): 
    6.18      op = req.get_options().get(OPTION_ALLOW_GROUP_WRITE) 
    6.19      return sys.platform != 'win32' \
    6.20 @@ -39,16 +44,19 @@
    6.21      base = tramline_path(req)
    6.22      for i, p in enumerate((tramline_path(req), 
    6.23                             tramline_upload_path(req), 
    6.24 +                           tramline_stat_path(req), 
    6.25                             tramline_repository_path(req))):
    6.26          if not os.path.isdir(p):
    6.27              os.mkdir(p)
    6.28 -            if i == 2 and group_write(req):
    6.29 +            import sys
    6.30 +            sys.stderr.write('creating path: %r\n' % p)
    6.31 +	    if i == 3 and group_write(req):
    6.32                  # drwxrwsr-x
    6.33                  # (set-group-ID-on-execution bit)
    6.34                  os.chmod(p, 02775)
    6.35  
    6.36  FILE_CHUNKSIZE = 8 * 1024
    6.37 -
    6.38 +REPORT_BLOCKSIZE = 100 * 1024
    6.39  """
    6.40  inputfilter() and outputfilter() are what is called by Apache.
    6.41  
    6.42 @@ -79,7 +87,8 @@
    6.43          pass_on(filter)
    6.44          return
    6.45  
    6.46 -    if filter.req.method == 'GET':
    6.47 +    req_method = filter.req.method
    6.48 +    if req_method == 'GET':
    6.49          hin = filter.req.headers_in
    6.50          range = hin.get('Range')
    6.51          if range is not None:
    6.52 @@ -87,16 +96,27 @@
    6.53              # hoping that most backing apps issue 206, not 200 for that
    6.54              hin['Range'] = 'bytes=0-' 
    6.55  
    6.56 -    # we only handle POST requests
    6.57 -    if filter.req.method != 'POST':
    6.58 +    
    6.59 +    # we only affect POST and PUT requests from now on
    6.60 +    if filter.req.method not in ('POST', 'PUT'):
    6.61          pass_on(filter)
    6.62          return
    6.63  
    6.64 -    # we only handle multipart/form-data
    6.65 -    enctype = filter.req.headers_in.get('Content-Type')
    6.66 -    if enctype[:19] != 'multipart/form-data':
    6.67 -        pass_on(filter)
    6.68 -        return
    6.69 +    if req_method == 'POST':
    6.70 +        # we only handle multipart/form-data
    6.71 +        enctype = filter.req.headers_in.get('Content-Type')
    6.72 +        if enctype[:19] != 'multipart/form-data':
    6.73 +            pass_on(filter)
    6.74 +            return
    6.75 +    elif req_method == 'PUT':
    6.76 +        # we only handle request that have the 'X-Tramline-Enable' header
    6.77 +        # or 'tramline_enable' cookie. 
    6.78 +        # Cookie enabling is here to support those agents that
    6.79 +        # can't set a custom header (e.g., zopeedit)
    6.80 +        if filter.req.headers_in.get('X-Tramline-Enable') is None \
    6.81 +                and Cookie.get_cookie(filter.req, 'tramline_enable') is None:
    6.82 +            pass_on(filter)
    6.83 +            return
    6.84  
    6.85      # check whether we have an id already
    6.86      id = filter.req.headers_in.get('tramline_id')
    6.87 @@ -104,7 +124,8 @@
    6.88      if id is None:
    6.89          # no id, so create new processor instance and store
    6.90          # away id
    6.91 -        processor = theProcessorRegistry.createProcessor()
    6.92 +        processor = theProcessorRegistry.createProcessor(req_method=req_method)
    6.93 +        processor.initFromInputFilter(filter)
    6.94          filter.req.headers_in['tramline_id'] = str(processor.id)
    6.95      else:
    6.96          # reuse existing processor instance based on id
    6.97 @@ -144,7 +165,7 @@
    6.98  
    6.99      # in case of post request, we may need to do a commit/abort
   6.100      # of previous input round
   6.101 -    if filter.req.method == 'POST':
   6.102 +    if filter.req.method in ('POST', 'PUT'):
   6.103          outputfilter_post(filter)
   6.104          return
   6.105      # in case of a get request, we may need to serve up files,
   6.106 @@ -198,18 +219,24 @@
   6.107      while s:
   6.108          data.append(s)
   6.109          s = filter.read()
   6.110 -    file_id = ''.join(data)
   6.111 +    data = ''.join(data)
   6.112 +
   6.113 +    # if multiple lines, the id is the last, first lines have to be forwarded
   6.114 +    split = data.rsplit('\n', 1)
   6.115 +    if len(split) == 1:
   6.116 +        file_id = data
   6.117 +        prepend = ''
   6.118 +    else:
   6.119 +        file_id = split[1]
   6.120 +        prepend = split[0] + '\n'
   6.121 +
   6.122      p = id_to_path(tramline_path(filter.req), file_id)
   6.123  
   6.124 -    log('file id:' + file_id, filter.req)
   6.125      if not file_id:
   6.126          return
   6.127      # Range request
   6.128      hin = filter.req.headers_in
   6.129      h = hin.get(TRAMLINE_RANGE_HEADER)
   6.130 -    log('Filter out: Range: ' + str(h), filter.req)
   6.131 -    #TODO log("Filter out: status line " + filter.req.status_line, filter.req)
   6.132 -    log("Filter out: status " + str(status), filter.req)
   6.133  
   6.134      if h and status in [206, 416]:
   6.135          # Range was requested and app server agreed
   6.136 @@ -219,13 +246,15 @@
   6.137  	else:
   6.138             log("Unparsable Range header " + h, filter.req)
   6.139      elif status == 200:
   6.140 -	serve_file(p, filter)
   6.141 +	serve_file(p, filter, prepend=prepend)
   6.142  
   6.143 -def serve_file(p, filter):
   6.144 +def serve_file(p, filter, prepend=''):
   6.145      """Serve a whole file."""
   6.146      # XXX what if file doesn't exist? 404?
   6.147      size = os.stat(p).st_size
   6.148 -    filter.req.headers_out['content-length'] = str(size)
   6.149 +    if prepend:
   6.150 +        filter.write(prepend)
   6.151 +    filter.req.headers_out['content-length'] = str(size + len(prepend))
   6.152      f = open(p, 'rb')
   6.153      dump_file_range(f, 0, size-1, filter)
   6.154      f.close()
   6.155 @@ -252,7 +281,6 @@
   6.156               filter.close()
   6.157               return
   6.158  
   6.159 -	log("serve_file_ranges: one chunk, %d-%d" % (start, end), filter.req)
   6.160    	hout['Content-Length'] = str(end - start + 1)
   6.161          hout['Content-Range'] = 'bytes %d-%d/%d' % (start, end, size)
   6.162  	dump_file_range(fd, start, end, filter)
   6.163 @@ -307,32 +335,137 @@
   6.164  
   6.165      
   6.166  class ProcessorRegistry:
   6.167 +
   6.168 +    _processor_classes = {}
   6.169 +
   6.170      def __init__(self):
   6.171          self._processors = {}
   6.172  
   6.173      def getProcessor(self, id):
   6.174          return self._processors[id]
   6.175  
   6.176 -    def createProcessor(self):
   6.177 +    def createProcessor(self, req_method='POST'):
   6.178          # XXX thread issues?
   6.179          while True:
   6.180              id = random.randrange(sys.maxint)
   6.181              if id not in self._processors:
   6.182                  break
   6.183 -        result = self._processors[id] = Processor(id)
   6.184 -        return result
   6.185  
   6.186 +        ProcessorClass = self._processor_classes.get(req_method)
   6.187 +        proc = self._processors[id] = ProcessorClass(id)
   6.188 +        return proc
   6.189 +    
   6.190      def removeProcessor(self, processor):
   6.191          del self._processors[processor.id]
   6.192  
   6.193  theProcessorRegistry = ProcessorRegistry()
   6.194  
   6.195 +def get_progress_path(req, progress_id):
   6.196 +    return os.path.join(tramline_stat_path(req), progress_id)
   6.197 +
   6.198  class Processor:
   6.199 +    """Base class for processors.
   6.200 +
   6.201 +    API atributes:
   6.202 +       id: this is the main identifier in the registry. It is used for 
   6.203 +           continuity from one call to the other and from input to ouput 
   6.204 +           filters (final commit).
   6.205 +
   6.206 +       progress_id: this is the identifier user agents will use to request 
   6.207 +           stats on the current transfer upload. Unfortunately it has to be
   6.208 +           partially set by an <input> from the user agent, because we have no
   6.209 +           means to communicate it back once the process has started. 
   6.210 +           Still some effort is made to avoid malicious garbling of ids.
   6.211 +
   6.212 +       uploaded: this is the total file data upload this processor has seen. 
   6.213 +           there might be more than one file in this request.
   6.214 +
   6.215 +       upload_length: the full length of the input request, as taken from the
   6.216 +       header
   6.217 +    """
   6.218      def __init__(self, id):
   6.219 +        self.uploaded = 0L
   6.220          self.id = id
   6.221 +        self.progress_id = None
   6.222 +        self._uploaded_blocks = 0 
   6.223          self._upload_files = []
   6.224          self._incoming = []
   6.225  	self._isize = 0
   6.226 +        self._f = None # output file object
   6.227 +
   6.228 +    def initFromInputFilter(self, filter):
   6.229 +        headers = filter.req.headers_in
   6.230 +        self.upload_length = long(headers['Content-Length'])
   6.231 +	qs = filter.req.parsed_uri[apache.URI_QUERY]
   6.232 +	if qs is not None:
   6.233 +           gu_ids = cgi.parse_qs(qs).get('gp.fileupload.id')
   6.234 +           if gu_ids is not None:
   6.235 +              # parse_qs always produces lists
   6.236 +              self.progress_id = filter.req.connection.remote_ip + '-' + gu_ids[0]
   6.237 +
   6.238 +    def logProgress(self, req):
   6.239 +        """Log progress upload to a file for async requests to use
   6.240 +
   6.241 +        Report only every REPORT_BLOCKSIZE bytes to avoid too much I/O
   6.242 +        the file is located in the stat directory, and its name is progress_id.
   6.243 +        Race condition is in theory possible, but very unlikely, and not
   6.244 +        critical to avoid.
   6.245 +        """
   6.246 +        blocks = self.uploaded / REPORT_BLOCKSIZE
   6.247 +        if blocks > self._uploaded_blocks:
   6.248 +            self._uploaded_blocks = blocks
   6.249 +            self._storeProgress(req)
   6.250 +
   6.251 +    def _storeProgress(self, req):
   6.252 +        if self.progress_id is not None:
   6.253 +            f = open(get_progress_path(req, self.progress_id), 'w')
   6.254 +            percent = int(self.uploaded*100 / self.upload_length)
   6.255 +            f.write(str({'state': 1, 'percent': int(percent)}))
   6.256 +            f.close()
   6.257 +
   6.258 +    def commit(self, req):
   6.259 +        # XXX works under the assumption that the last segment of 
   6.260 +        # file path is the tramline id
   6.261 +        for upload_file in self._upload_files:
   6.262 +            dummy, filename = os.path.split(upload_file)
   6.263 +            os.rename(upload_file, id_to_path(tramline_path(req), filename))
   6.264 +
   6.265 +    def abort(self):
   6.266 +        for upload_file in self._upload_files:
   6.267 +            os.remove(upload_file)
   6.268 +
   6.269 +    def initUploadFile(self, out, with_newline=True):
   6.270 +        """Create the dump file, write id to out and keep needed references."""
   6.271 +        fd, pathname, file_id = createUniqueFile(out.req)
   6.272 +
   6.273 +        self._f = os.fdopen(fd, 'wb')
   6.274 +        self._upload_files.append(pathname)
   6.275 +        out.write(file_id)
   6.276 +        if with_newline:
   6.277 +            out.write('\r\n')
   6.278 +
   6.279 +class PutRequestProcessor(Processor):
   6.280 +   """Directly dump inconditionaly incoming data."""
   6.281 +
   6.282 +   def pushInput(self, data, out):
   6.283 +       if self._f is None:
   6.284 +           self.initUploadFile(out, with_newline=False)
   6.285 +       self._f.write(data)
   6.286 +
   6.287 +   def finalizeInput(self, out):
   6.288 +       out.req.headers_in['tramline'] = ''
   6.289 +       if self._f is not None: # one never knows
   6.290 +           self._f.close()
   6.291 +
   6.292 +class PostRequestProcessor(Processor):
   6.293 +    """Processor for Post requests. 
   6.294 +
   6.295 +    Handles multipart/form-data content, looks for enabling info in the form
   6.296 +    data, and if enabled, intercept those parts that are file uploads.
   6.297 +    """
   6.298 +
   6.299 +    def __init__(self, pid):
   6.300 +        Processor.__init__(self, pid)
   6.301          # we use a state pattern where the handle method gets
   6.302          # replaced by the current handle method for this state.
   6.303          self.handle = self.handle_first_boundary
   6.304 @@ -349,7 +482,7 @@
   6.305          self._incoming.append(data)
   6.306  	self._isize += len(data)
   6.307  
   6.308 -        # if we're not at the end of the line, input was broken
   6.309 +       # if we're not at the end of the line, input was broken
   6.310          # somewhere, unless we are handling file data which might be binary.
   6.311  	# We return to collect more first (also for file data if too small)
   6.312          if data[-1] != '\n' and (
   6.313 @@ -365,22 +498,12 @@
   6.314          self._isize = 0
   6.315  
   6.316          self.handle(line, out)
   6.317 +        self.logProgress(out.req)
   6.318  
   6.319      def finalizeInput(self, out):
   6.320          if self._upload_files:
   6.321              out.req.headers_in['tramline'] = ''
   6.322  
   6.323 -    def commit(self, req):
   6.324 -        # XXX works under the assumption that the last segment of 
   6.325 -        # file path is the tramline id
   6.326 -        for upload_file in self._upload_files:
   6.327 -            dummy, filename = os.path.split(upload_file)
   6.328 -            os.rename(upload_file, id_to_path(tramline_path(req), filename))
   6.329 -
   6.330 -    def abort(self):
   6.331 -        for upload_file in self._upload_files:
   6.332 -            os.remove(upload_file)
   6.333 -    
   6.334      def handle_first_boundary(self, line, out):
   6.335          self._boundary = line
   6.336          self._last_boundary = self._boundary.rstrip() + '--\r\n'
   6.337 @@ -412,21 +535,22 @@
   6.338          filename = self._disposition_options.get('filename')
   6.339          # if filename is empty, assume no file is submitted and submit
   6.340          # empty file -- don't tramline this special case
   6.341 -        if out.req.get_options().get('explicit_enable') and \
   6.342 -              self._disposition_options.get('name')=='tramline_enable':
   6.343 +        input_name = self._disposition_options.get('name')
   6.344 +        if input_name == 'tramline_enable' and out.req.get_options().get(
   6.345 +            'explicit_enable'):
   6.346              self.handle = self.handle_enable_vars
   6.347              return
   6.348 +        elif input_name == 'tramline_progress_id':
   6.349 +            # identifier used by async user agent requests to get upload stats
   6.350 +            self.handle = self.handle_progress_id
   6.351 +            return
   6.352          elif (filename is None or not filename) or \
   6.353                out.req.get_options().get('explicit_enable') and \
   6.354                self._disposition_options.get('name') not in self.vars_to_handle:
   6.355              self.handle = self.handle_data
   6.356              return
   6.357 -        fd, pathname, file_id = createUniqueFile(out.req)
   6.358  
   6.359 -        self._f = os.fdopen(fd, 'wb')
   6.360 -        self._upload_files.append(pathname)
   6.361 -        out.write(file_id)
   6.362 -        out.write('\r\n')
   6.363 +        self.initUploadFile(out)
   6.364          
   6.365          self._previous_line = None
   6.366          self.handle = self.handle_file_data
   6.367 @@ -446,6 +570,22 @@
   6.368          else:
   6.369              self._enable_vars+=line
   6.370  
   6.371 +    def handle_progress_id(self, line, out):
   6.372 +        out.write(line)
   6.373 +        if line == self._boundary:
   6.374 +            self.init_headers()
   6.375 +            self.handle = self.handle_headers
   6.376 +        elif line == self._last_boundary:
   6.377 +            # shouldn't happen if client has some consistency 
   6.378 +            self.handle = None # Processing done
   6.379 +        else:
   6.380 +            # GR: full socket info would be preferable, but it'll be different
   6.381 +            # for stat requests, and not sure the user agent can know and store
   6.382 +            # the full socket info of the upload for requesting.
   6.383 +            self.progress_id = out.req.connection.remote_ip + '-' + line.strip()
   6.384 +            # initialisation of the store
   6.385 +            self._storeProgress(out.req)
   6.386 +
   6.387      def handle_data(self, line, out):
   6.388          out.write(line)
   6.389          if line == self._boundary:
   6.390 @@ -459,6 +599,7 @@
   6.391          if line == self._boundary:
   6.392              # write last line, but without \r\n
   6.393              self._f.write(self._previous_line[:-2])
   6.394 +            self.uploaded += len(self._previous_line[:-2])
   6.395              out.write(line)
   6.396              self._f.close()
   6.397              self._f = None
   6.398 @@ -466,6 +607,7 @@
   6.399          elif line == self._last_boundary:
   6.400              # write last line, but without \r\n
   6.401              self._f.write(self._previous_line[:-2])
   6.402 +            self.uploaded += len(self._previous_line[:-2]) # for completeness
   6.403              out.write(line)
   6.404              self._f.close()
   6.405              self._f = None
   6.406 @@ -473,8 +615,18 @@
   6.407          else:
   6.408              if self._previous_line is not None:
   6.409                  self._f.write(self._previous_line)
   6.410 +                self.uploaded += len(self._previous_line)
   6.411              self._previous_line = line
   6.412  
   6.413 +def get_progress(req, progress_id):
   6.414 +    try:
   6.415 +        f = open(get_progress_path(req, progress_id), 'r')
   6.416 +    except IOError: # inaccessible progress log: process not really started
   6.417 +        return str({'state': 0, 'percent': 0})
   6.418 +    s = f.read()
   6.419 +    f.close() # let's be explicit :-)
   6.420 +    return s
   6.421 +
   6.422  def parse_header(s):
   6.423      l = [e.strip() for e in s.split(';')]
   6.424      result_value = l.pop(0).lower()
   6.425 @@ -519,6 +671,9 @@
   6.426                  continue # try again
   6.427              raise
   6.428  
   6.429 +ProcessorRegistry._processor_classes = dict(PUT=PutRequestProcessor,
   6.430 +                                            POST=PostRequestProcessor)
   6.431 +
   6.432  def log(data, req):
   6.433      f = open(os.path.join(tramline_path(req), 'tramline.log'), 'ab')
   6.434      f.write(data)
     7.1 new file mode 100644
     7.2 --- /dev/null
     7.3 +++ b/src/tramline/progress.py
     7.4 @@ -0,0 +1,12 @@
     7.5 +from mod_python import apache
     7.6 +from tramline.core import get_progress
     7.7 +
     7.8 +def handler(req):
     7.9 +    progress_id = req.connection.remote_ip + '-' + req.uri.rsplit('/', 1)[-1]
    7.10 +    progress = get_progress(req, progress_id)
    7.11 +
    7.12 +    req.content_type = "text/plain"
    7.13 +    req.headers_out['Content-length'] = str(len(progress))
    7.14 +    req.write(progress)
    7.15 +
    7.16 +    return apache.OK
     8.1 new file mode 100644
     8.2 --- /dev/null
     8.3 +++ b/src/tramline/tests/data/input8.txt
     8.4 @@ -0,0 +1,16 @@
     8.5 +-----------------------------100323068321119442571506749230
     8.6 +Content-Disposition: form-data; name="tramline_progress_id"
     8.7 +
     8.8 +1234
     8.9 +-----------------------------100323068321119442571506749230
    8.10 +Content-Disposition: form-data; filename="test.txt"; name="test"
    8.11 +Content-Type: application/octet-stream
    8.12 +
    8.13 +first line
    8.14 +second line
    8.15 +
    8.16 +-----------------------------100323068321119442571506749230
    8.17 +Content-Disposition: form-data; name="submit"
    8.18 +
    8.19 +submit data
    8.20 +-----------------------------100323068321119442571506749230--
     9.1 --- a/src/tramline/tests/test_core.py
     9.2 +++ b/src/tramline/tests/test_core.py
     9.3 @@ -9,6 +9,11 @@
     9.4  from tramline.core import OPTION_ALLOW_GROUP_WRITE
     9.5  from tramline.core import TRAMLINE_RANGE_HEADER
     9.6  
     9.7 +from tramline.core import theProcessorRegistry
     9.8 +
     9.9 +from tramline import core as tramcore
    9.10 +tramcore.REPORT_BLOCKSIZE = 5
    9.11 +
    9.12  tramline_path = '/tmp/trampath'
    9.13  
    9.14  class StringTable(dict):
    9.15 @@ -17,13 +22,18 @@
    9.16              raise ValueError("Table values must be strings")
    9.17          dict.__setitem__(self, key, value)
    9.18  
    9.19 +class Connection:
    9.20 +    remote_ip = 'TESTIP'
    9.21 +
    9.22  class Request:
    9.23      def __init__(self, method):
    9.24 -        self.headers_in = StringTable({'Content-Type' : 'multipart/form-data'})
    9.25 +        self.headers_in = StringTable({'Content-Type' : 'multipart/form-data',
    9.26 +                                       'Content-Length': '56'})
    9.27          self.headers_out = StringTable()
    9.28          self.main = None
    9.29          self.method = method
    9.30          self.options = {'tramline_path': tramline_path}
    9.31 +        self.connection = Connection()
    9.32  
    9.33      def get_options(self):
    9.34          return self.options
    9.35 @@ -136,6 +146,38 @@
    9.36          self.assertEquals(
    9.37              'first line\nsecond line', data)
    9.38          
    9.39 +    def test_inputfilter_progress_id(self):
    9.40 +        input = open(get_data_path('input8.txt'), 'rb')
    9.41 +        progress_id = 'TESTIP-1234'
    9.42 +        output = StringIO()
    9.43 +        filter = Filter(input, output)
    9.44 +        
    9.45 +        inputfilter(filter)
    9.46 +
    9.47 +        input.close()
    9.48 +
    9.49 +        from tramline.core import get_progress
    9.50 +        progress = get_progress(filter.req, progress_id)
    9.51 +        self.assertFalse(progress is None)
    9.52 +
    9.53 +        output_data = output.getvalue()
    9.54 +
    9.55 +        file_id = self.file_id(output_data)
    9.56 +        f = open(os.path.join(tramline_upload_path(filter.req), file_id), 'rb')
    9.57 +        
    9.58 +        data = f.read()
    9.59 +        f.close()
    9.60 +        expected = 'first line\nsecond line\n'
    9.61 +        self.assertEquals(23, len(expected)) # update this if needed
    9.62 +        self.assertEquals(expected, data)
    9.63 +        # process has finished, so uploaded should be the entire length
    9.64 +        # but the fake request is suppose to be 56 bytes long
    9.65 +        self.assertEquals("{'state': 1, 'percent': 41}", progress)
    9.66 +
    9.67 +        # Now requesting progress info with an unknown ID
    9.68 +        self.assertEquals("{'state': 0, 'percent': 0}",
    9.69 +                          get_progress(filter.req, 'TESTIP-4567'))
    9.70 +        
    9.71      def test_split_filter(self):
    9.72          f = open(get_data_path('input2.txt'), 'rb')
    9.73          data = f.read()