vendor/tramline

changeset 0:9d9f6733500e gracinet-fix-range

#1919: patched version of tramline fixes pass_on() bug and implement Range HTTP requests
author gracinet
date Sun, 12 Oct 2008 22:31:10 +0000
parents
children 00f158ab38ae
files CHANGES CREDITS.txt HISTORY INSTALL.txt LICENSE.txt README.txt VERSION doc/licenses/BSD.txt doc/presentation.txt setup.py src/tramline/__init__.py src/tramline/core.py src/tramline/headers.py src/tramline/interfaces.py src/tramline/simplefilter.py src/tramline/tests/__init__.py src/tramline/tests/data/input1.txt src/tramline/tests/data/input2.txt src/tramline/tests/data/input3.txt src/tramline/tests/data/input4.txt src/tramline/tests/data/input5.txt src/tramline/tests/data/input6.txt src/tramline/tests/data/input7.txt src/tramline/tests/data/output7_ranges.txt src/tramline/tests/test_core.py
diffstat 25 files changed, 1691 insertions(+), 0 deletions(-) [+]
line diff
     1.1 new file mode 100644
     1.2 --- /dev/null
     1.3 +++ b/CHANGES
     1.4 @@ -0,0 +1,12 @@
     1.5 +Requires
     1.6 +~~~~~~~~
     1.7 +-
     1.8 +New features
     1.9 +~~~~~~~~~~~~
    1.10 +-
    1.11 +Bug fixes
    1.12 +~~~~~~~~~
    1.13 +-
    1.14 +New internal features
    1.15 +~~~~~~~~~~~~~~~~~~~~~
    1.16 +- 
     2.1 new file mode 100644
     2.2 --- /dev/null
     2.3 +++ b/CREDITS.txt
     2.4 @@ -0,0 +1,24 @@
     2.5 +Credits
     2.6 +=======
     2.7 +
     2.8 +Tramline was originally developed by Infrae (http://www.infrae.com).
     2.9 +
    2.10 +Developers
    2.11 +----------
    2.12 +
    2.13 +* Martijn Faassen (concept, developer)
    2.14 +
    2.15 +* Jan-Wijbrand Kolman (concept, developer)
    2.16 +
    2.17 +* Emyr Thomas (bugfixes, testing with Plone)
    2.18 +
    2.19 +* Luis De La Parra (bugfixes, testing with Plone)
    2.20 +
    2.21 +* Daniel Nouri (smallish improvements)
    2.22 +
    2.23 +* Chad Maine (bug reporting)
    2.24 +
    2.25 +* Jeroen Vloothuis (explicit enabling feature)
    2.26 +
    2.27 +Thanks also to Guido Goldstein for forging Railroad, tramline's big
    2.28 +brother.
     3.1 new file mode 100644
     3.2 --- /dev/null
     3.3 +++ b/HISTORY
     3.4 @@ -0,0 +1,20 @@
     3.5 +===========================================================
     3.6 +Package: tramline-viral 0.9.0
     3.7 +===========================================================
     3.8 +First release built by: gracinet at: 2008-10-09T13:17:43
     3.9 +SVN Tag: https://viral-prod.com/svn/tramline/tags/0.9.0
    3.10 +Build from: https://viral-prod.com/svn/tramline/trunk@939
    3.11 +
    3.12 +Requires
    3.13 +~~~~~~~~
    3.14 +-
    3.15 +New features
    3.16 +~~~~~~~~~~~~
    3.17 +-
    3.18 +Bug fixes
    3.19 +~~~~~~~~~
    3.20 +-
    3.21 +New internal features
    3.22 +~~~~~~~~~~~~~~~~~~~~~
    3.23 +- First package
    3.24 +
     4.1 new file mode 100644
     4.2 --- /dev/null
     4.3 +++ b/INSTALL.txt
     4.4 @@ -0,0 +1,55 @@
     4.5 +Installing tramline
     4.6 +===================
     4.7 +
     4.8 +Requirements
     4.9 +------------
    4.10 +
    4.11 +* It needs Apache 2.0.55 or higher. Apache 2.0.54 and below do *not*
    4.12 +  work due to a bug in mod_proxy filter handling.
    4.13 +
    4.14 +* You tend to need mod_proxy installed to proxy requests that come
    4.15 +  into Apache into your underlying appserver. It also works with
    4.16 +  mod_rewrite if you use proxying mode for this ([P]).
    4.17 +
    4.18 +* mod_python, with the apache.py file patched so filter is not
    4.19 +  flushed. In mod_python/lib/python/apache.py, comment out
    4.20 +  'filter.flush()' in FilterDispatch.
    4.21 +
    4.22 +Apache conf
    4.23 +-----------
    4.24 +
    4.25 +* Enable mod_python.
    4.26 +
    4.27 +* Then:
    4.28 +
    4.29 +  PythonPath "sys.path+['/path/to/tramline/src']"
    4.30 +  PythonInputFilter tramline.core::inputfilter TRAMLINE_INPUT
    4.31 +  PythonOutputFilter tramline.core::outputfilter TRAMLINE_OUTPUT
    4.32 +  SetInputFilter TRAMLINE_INPUT
    4.33 +  SetOutputFilter TRAMLINE_OUTPUT
    4.34 +  PythonOption tramline_path /path/to/tramline-storage
    4.35 +
    4.36 +  The last line defines where files should be put on your filesystem.
    4.37 +
    4.38 +  Note that if tramline is installed as a Python package into the same
    4.39 +  Python installation as the one mod_python uses, the PythonPath line
    4.40 +  can go away.
    4.41 +
    4.42 +Example: Zope
    4.43 +-------------
    4.44 +
    4.45 +This is an example configuration of how to run tramline for
    4.46 +*development purposes* with a local Apache and a local Zope 3.
    4.47 +
    4.48 +* Make sure you have your Apache configured with 'rewrite' support.
    4.49 +  An example:
    4.50 +
    4.51 +  ./configure --prefix=~/lib/apache2 --enable-rewrite
    4.52 +
    4.53 +* In addition to what's described in the 'Apache Conf', you need to
    4.54 +  set up your Rewrite rule.  I assume that you have Zope 3 running
    4.55 +  locally on 8080 and Apache running on 8000:
    4.56 +
    4.57 +  RewriteEngine On
    4.58 +  RewriteRule ^/(/?.*) \
    4.59 +    http://localhost:8080/++vh++http:localhost:8000/++/$1 [P,L]
     5.1 new file mode 100644
     5.2 --- /dev/null
     5.3 +++ b/LICENSE.txt
     5.4 @@ -0,0 +1,2 @@
     5.5 +tramline is copyright Infrae and distributed under the BSD license
     5.6 +(see doc/licenses/BSD.txt), with the following exceptions:
     6.1 new file mode 100644
     6.2 --- /dev/null
     6.3 +++ b/README.txt
     6.4 @@ -0,0 +1,83 @@
     6.5 +Tramline
     6.6 +========
     6.7 +
     6.8 +Introduction
     6.9 +------------
    6.10 +
    6.11 +Tramline is a upload and download accelerator that plugs into Apache,
    6.12 +using mod_python. Its aim is to make downloading and uploading large
    6.13 +media to an application server easy and fast, without overloading the
    6.14 +application server with large amounts of binary data.
    6.15 +
    6.16 +Tramline integrates into Apache using mod_python. The application
    6.17 +server is assumed to sit behind Apache, for instance hooked up using
    6.18 +mod_proxy or mod_rewrite.
    6.19 +
    6.20 +Tramline takes over uploading and downloading files, handling these
    6.21 +within Apache. Only a small configuration change in Apache should be
    6.22 +necessary to enable tramline.
    6.23 +
    6.24 +The application server remains in complete control over security, page
    6.25 +and form rendering, and everything else. Minimal changes are necessary
    6.26 +to any application to enable it to work with tramline; in fact it's
    6.27 +just setting two response headers in a few places in the code.
    6.28 +
    6.29 +How it works
    6.30 +------------
    6.31 +
    6.32 +Given a 'tramline_data' directory that's accessible to Apache (and the
    6.33 +appserver if it needs to), there are two subdirectories, 'upload' and
    6.34 +'repository'. 'upload' will only contain temporary files currently
    6.35 +being uploaded, while 'repository' contains the files successfully
    6.36 +uploaded.
    6.37 +
    6.38 +Tramline makes sure uploaded files (in a form POST) don't appear at
    6.39 +the appserver but go directly into the filesystem. The only thing the
    6.40 +appserver sees is a unique identifier of the uploaded file, so that
    6.41 +the appserver can access it when needed. The binary data is gone at
    6.42 +the time the POST reaches the appserver. You can check whether
    6.43 +tramline is in use by checking the 'tramline' header in the request,
    6.44 +though frequently there's no need to do so.
    6.45 +
    6.46 +The appserver can control whether it accepts the uploaded file(s) in
    6.47 +the output response header; if a 'tramline_ok' header is present, the
    6.48 +uploaded files will be moved into the repository, 'committing' the
    6.49 +upload. If it's absent, the uploaded files will be removed, 'aborting'
    6.50 +the upload.
    6.51 +
    6.52 +Tramline also can handle downloads. The appserver can signal in the
    6.53 +response headers that tramline should push a file out of the
    6.54 +filesystem to the end user, by adding a 'tramline_file' response
    6.55 +header. The data of the file body as received during upload,
    6.56 +containing the unique identifier of the file, should be sent back as
    6.57 +the response body.  Again the appserver does not see the binary data
    6.58 +but only sends out an identifier to make the file be served by Apache.
    6.59 +
    6.60 +Tramline makes it relatively easy to make an application that handled
    6.61 +large file uploads correctly without tramline installed as well. After
    6.62 +all, the application handles a tramline id just like it would handle
    6.63 +an uploaded file; the data is stored and served again. Of course
    6.64 +mixing tramline uploaded files and appserver uploaded files in the
    6.65 +same setup of your application would get complicated, but this feature
    6.66 +does make it nice to be able to test your application without tramline
    6.67 +available.
    6.68 +
    6.69 +So:
    6.70 +
    6.71 +* to handle upload:
    6.72 +
    6.73 +  - file contents will contain the unique file id.
    6.74 +
    6.75 +  - send out 'tramline_ok' header if file is accepted. Failure 
    6.76 +    to send out this header will cause the file to be rejected.
    6.77 +
    6.78 +* to detect whether tramline took care of an upload:
    6.79 +
    6.80 +  - look for a 'tramline' header in the request.
    6.81 +
    6.82 +* to handle download:
    6.83 +
    6.84 +  - send out 'tramline_file' header in response if the file can
    6.85 +    be downloaded.
    6.86 +
    6.87 +  - send out response body with unique file id.
     7.1 new file mode 100644
     7.2 --- /dev/null
     7.3 +++ b/VERSION
     7.4 @@ -0,0 +1,5 @@
     7.5 +# BUNDLEMAN PRODUCT CONFIGURATION FILE
     7.6 +# do not edit this file
     7.7 +PKG_NAME=tramline-viral
     7.8 +PKG_VERSION=0.9.0
     7.9 +PKG_RELEASE=1
     8.1 new file mode 100644
     8.2 --- /dev/null
     8.3 +++ b/doc/licenses/BSD.txt
     8.4 @@ -0,0 +1,29 @@
     8.5 +Copyright (c) 2005 Infrae. All rights reserved.
     8.6 +
     8.7 +Redistribution and use in source and binary forms, with or without
     8.8 +modification, are permitted provided that the following conditions are
     8.9 +met:
    8.10 +
    8.11 +  1. Redistributions of source code must retain the above copyright
    8.12 +     notice, this list of conditions and the following disclaimer.
    8.13 +   
    8.14 +  2. Redistributions in binary form must reproduce the above copyright
    8.15 +     notice, this list of conditions and the following disclaimer in
    8.16 +     the documentation and/or other materials provided with the
    8.17 +     distribution.
    8.18 +
    8.19 +  3. Neither the name of Infrae nor the names of its contributors may
    8.20 +     be used to endorse or promote products derived from this software
    8.21 +     without specific prior written permission.
    8.22 +
    8.23 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
    8.24 +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
    8.25 +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
    8.26 +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INFRAE OR
    8.27 +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
    8.28 +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
    8.29 +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
    8.30 +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
    8.31 +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
    8.32 +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
    8.33 +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     9.1 new file mode 100644
     9.2 --- /dev/null
     9.3 +++ b/doc/presentation.txt
     9.4 @@ -0,0 +1,112 @@
     9.5 +Tramline
     9.6 +========
     9.7 +
     9.8 +This presentation is based on a lightning talk at the Plone Conference
     9.9 +2005 in Vienna by Martijn Faassen. It has since been updated.
    9.10 +
    9.11 +Introduction
    9.12 +------------
    9.13 +
    9.14 +* Upload and download accelerator for Zope (2 or 3)
    9.15 +
    9.16 +* Plugs into the Apache HTTP server.
    9.17 +
    9.18 +* Large file data not stored in the ZODB
    9.19 +
    9.20 +* Railroad's little sister
    9.21 +
    9.22 +Why tramline?
    9.23 +-------------
    9.24 +
    9.25 +* railroad powerful repository
    9.26 +
    9.27 +* WebDAV, metadata, multi-CMS, search
    9.28 +
    9.29 +* but complicated integration with applications
    9.30 +
    9.31 +Tramline Goals
    9.32 +--------------
    9.33 +
    9.34 +* Deliberately limited goals
    9.35 +
    9.36 +* Fast uploads and downloads of files
    9.37 +
    9.38 +* Store large file data in filesystem, not ZODB
    9.39 +
    9.40 +* Easy to integrate with any application
    9.41 +
    9.42 +* No need to cut down forests for a tramline
    9.43 +
    9.44 +Design
    9.45 +------
    9.46 +
    9.47 +* Plugs into Apache with mod_python
    9.48 +
    9.49 +* Simple setup in Apache
    9.50 +
    9.51 +* Works with any form that uploads files
    9.52 +
    9.53 +* Need two one-liners in your applications to make it work
    9.54 +
    9.55 +Upload one-liner
    9.56 +----------------
    9.57 +
    9.58 +In response header to upload form, send out 'tramline_ok' header (no
    9.59 +contents)::
    9.60 +
    9.61 +  response.addHeader('tramline_ok', '')
    9.62 +
    9.63 +If that's not sent, tramline throws away the file. So, if application
    9.64 +raises error, user is unauthored, etc, file is thrown away.
    9.65 +
    9.66 +Upload continued
    9.67 +----------------
    9.68 +
    9.69 +* File received by the server is just an id string, very
    9.70 +  short, ZODB can scale to store these easily. :)
    9.71 +
    9.72 +* Can store this just as any other file
    9.73 +
    9.74 +Download
    9.75 +--------
    9.76 +
    9.77 +* serve up file (the id) by returning the content, just
    9.78 +  like any other file
    9.79 +
    9.80 +* Add a response header 'tramline_file' (another oneliner)
    9.81 +
    9.82 +Performance
    9.83 +-----------
    9.84 +
    9.85 +* Tested with chandler tarball, 26 megs of file
    9.86 +
    9.87 +* This is really fast
    9.88 +
    9.89 +Drawbacks
    9.90 +---------
    9.91 +
    9.92 +* Needs latest stable Apache (2.0.55) (mod_proxy inputfilter bugfixes
    9.93 +  needed)
    9.94 +
    9.95 +* Needs 1 line patch to mod_python
    9.96 +
    9.97 +* Reported inputfilter problem to mod_python developers
    9.98 +
    9.99 +Interesting benefit
   9.100 +-------------------
   9.101 +
   9.102 +* You can now do ridiculously unscalable things, scalably
   9.103 +
   9.104 +* Can stuff files in sessions!
   9.105 +
   9.106 +* Can base64 encode files on page!
   9.107 +
   9.108 +* hurry.file makes use of this property of tramline
   9.109 +
   9.110 +Conclusion
   9.111 +----------
   9.112 +
   9.113 +* Tramline is fast and easy
   9.114 +
   9.115 +* Use it when you need fast upload and download and filesystem storage
   9.116 +  of large files
    10.1 new file mode 100644
    10.2 --- /dev/null
    10.3 +++ b/setup.py
    10.4 @@ -0,0 +1,43 @@
    10.5 +from setuptools import setup, find_packages
    10.6 +
    10.7 +version = '0.6'
    10.8 +
    10.9 +setup(name='tramline',
   10.10 +      version=version,
   10.11 +      description=("An easy and fast upload and download accelerator "
   10.12 +                   "for application servers."),
   10.13 +      long_description="""\
   10.14 +Tramline is a upload and download accelerator that plugs into Apache,
   10.15 +using mod_python. Its aim is to make downloading and uploading large
   10.16 +media to an application server easy and fast, without overloading the
   10.17 +application server with large amounts of binary data.
   10.18 +
   10.19 +Tramline integrates into Apache using mod_python. The application
   10.20 +server is assumed to sit behind Apache, for instance hooked up using
   10.21 +mod_proxy or mod_rewrite.
   10.22 +
   10.23 +Tramline takes over uploading and downloading files, handling these
   10.24 +within Apache. Only a small configuration change in Apache should be
   10.25 +necessary to enable tramline.
   10.26 +
   10.27 +The application server remains in complete control over security, page
   10.28 +and form rendering, and everything else. Minimal changes are necessary
   10.29 +to any application to enable it to work with tramline; in fact it's
   10.30 +just setting two response headers in a few places in the code.
   10.31 +""",
   10.32 +      classifiers=[
   10.33 +        "Development Status :: 5 - Production/Stable",
   10.34 +        "Intended Audience :: Developers",
   10.35 +        "License :: OSI Approved :: BSD License",
   10.36 +        "Programming Language :: Python",
   10.37 +        "Topic :: Internet :: WWW/HTTP",
   10.38 +        ],
   10.39 +      keywords='web zope command-line skeleton project',
   10.40 +      author='Martijn Faassen',
   10.41 +      author_email='faassen@infrae.com',
   10.42 +      url='http://www.infrae.com/products/tramline',
   10.43 +      packages=find_packages('src'),
   10.44 +      package_dir = {'': 'src'},
   10.45 +      include_package_data=True,
   10.46 +      zip_safe=False,
   10.47 +      )
    11.1 new file mode 100644
    11.2 --- /dev/null
    11.3 +++ b/src/tramline/__init__.py
    11.4 @@ -0,0 +1,1 @@
    11.5 +# this is a package
    12.1 new file mode 100644
    12.2 --- /dev/null
    12.3 +++ b/src/tramline/core.py
    12.4 @@ -0,0 +1,527 @@
    12.5 +import os, tempfile, random, sys, errno, mimetools
    12.6 +
    12.7 +OPTION_ALLOW_GROUP_WRITE = 'allow_group_write'
    12.8 +TRAMLINE_RANGE_HEADER = 'X-Tramline-Original-Range'
    12.9 +
   12.10 +from headers import parse_range
   12.11 +
   12.12 +def pass_on(filter):
   12.13 +    try:
   12.14 +      s = filter.read()
   12.15 +      while s:
   12.16 +          filter.write(s)
   12.17 +          s = filter.read()
   12.18 +    except IOError:
   12.19 +       s = None 
   12.20 +    if s is None:
   12.21 +        try:
   12.22 +          filter.close()
   12.23 +        except IOError: pass 
   12.24 +
   12.25 +
   12.26 +def tramline_path(req):
   12.27 +    return req.get_options()['tramline_path']
   12.28 +
   12.29 +def tramline_upload_path(req):
   12.30 +    return os.path.join(tramline_path(req), 'upload')
   12.31 +
   12.32 +def tramline_repository_path(req):
   12.33 +    return os.path.join(tramline_path(req), 'repository')
   12.34 +
   12.35 +def group_write(req): 
   12.36 +    op = req.get_options().get(OPTION_ALLOW_GROUP_WRITE) 
   12.37 +    return sys.platform != 'win32' \
   12.38 +        and op is not None and op.strip().lower() == 'true' 
   12.39 +
   12.40 +def create_paths(req):
   12.41 +    base = tramline_path(req)
   12.42 +    for i, p in enumerate((tramline_path(req), 
   12.43 +                           tramline_upload_path(req), 
   12.44 +                           tramline_repository_path(req))):
   12.45 +        if not os.path.isdir(p):
   12.46 +            os.mkdir(p)
   12.47 +            if i == 2 and group_write(req):
   12.48 +                # drwxrwsr-x
   12.49 +                # (set-group-ID-on-execution bit)
   12.50 +                os.chmod(p, 02775)
   12.51 +
   12.52 +def id_to_path(tramline_path, id, upload=False, create_intermediate=False):
   12.53 +    """Compute the path on filesystem of the file with given id.
   12.54 +
   12.55 +    tramline_path: the tramline base path, as extracted from request options.
   12.56 +    id: the tramline id, as a string.
   12.57 +    upload: if True, the computed path will be from the "upload" directory
   12.58 +            instead of the "repository"
   12.59 +    create_intermediate: if True, all necessary objects will be created, but
   12.60 +        the target file won't."""
   12.61 +    
   12.62 +    return os.path.join(tramline_path, upload and "upload" or "repository",
   12.63 +                        id)
   12.64 +    
   12.65 +
   12.66 +FILE_CHUNKSIZE = 8 * 1024
   12.67 +
   12.68 +"""
   12.69 +inputfilter() and outputfilter() are what is called by Apache.
   12.70 +
   12.71 +There can potentially (theoretically? it's hard to reproduce) be
   12.72 +multiple calls per request in the case of large amount of data being
   12.73 +uploaded. read() and readline() are not guaranteed to deliver all
   12.74 +data, but may just stop in the middle of a request. .read() may then
   12.75 +return an empty string. We know the filtering is actually done when
   12.76 +filter.read() returns None.
   12.77 +
   12.78 +We make sure multiple calls to inputfilter and outputfilter in a
   12.79 +request end up together, i.e. the right FilterProcessor class
   12.80 +instance. We do this by tucking away a process-unique id inside the
   12.81 +request headers. This way we can identify we're actually a subsequent
   12.82 +call for an existing request request. The id is used to look up the
   12.83 +FilterProcessor class instance in the FilterRegistry and the data is
   12.84 +pushed into it.
   12.85 +
   12.86 +Care is taken so that not too much memory is used when reading
   12.87 +in data. Apache turns out to read in chunks of about 8 kilobytes
   12.88 +typically, and at most this amount of data is kept in memory at any
   12.89 +one point. Data is written out as soon as possible.
   12.90 +"""
   12.91 +
   12.92 +def inputfilter(filter):
   12.93 +    # we're done if we're in a subrequest
   12.94 +    if filter.req.main is not None:
   12.95 +        pass_on(filter)
   12.96 +        return
   12.97 +
   12.98 +    if filter.req.method == 'GET':
   12.99 +        hin = filter.req.headers_in
  12.100 +        range = hin.get('Range')
  12.101 +        if range is not None:
  12.102 +            hin[TRAMLINE_RANGE_HEADER] = range
  12.103 +            # hoping that most backing apps issue 206, not 200 for that
  12.104 +            hin['Range'] = 'bytes=0-' 
  12.105 +
  12.106 +    # we only handle POST requests
  12.107 +    if filter.req.method != 'POST':
  12.108 +        pass_on(filter)
  12.109 +        return
  12.110 +
  12.111 +    # we only handle multipart/form-data
  12.112 +    enctype = filter.req.headers_in.get('Content-Type')
  12.113 +    if enctype[:19] != 'multipart/form-data':
  12.114 +        pass_on(filter)
  12.115 +        return
  12.116 +
  12.117 +    # check whether we have an id already
  12.118 +    id = filter.req.headers_in.get('tramline_id')
  12.119 +
  12.120 +    if id is None:
  12.121 +        # no id, so create new processor instance and store
  12.122 +        # away id
  12.123 +        processor = theProcessorRegistry.createProcessor()
  12.124 +        filter.req.headers_in['tramline_id'] = str(processor.id)
  12.125 +    else:
  12.126 +        # reuse existing processor instance based on id
  12.127 +        processor = theProcessorRegistry.getProcessor(int(id))
  12.128 +        
  12.129 +    # read data from filter. Result can be a string, an empty string
  12.130 +    # or None.
  12.131 +    s = filter.read()
  12.132 +    while s:
  12.133 +        # as long as we have data, push it into processor
  12.134 +        processor.pushInput(s, filter)
  12.135 +        s = filter.read()
  12.136 +    # if we got no more data, this may mean we are broken in the middle
  12.137 +    # of a request. In that case, we're done for now, but inputfilter()
  12.138 +    # will be called again later
  12.139 +    if s is not None:
  12.140 +        return
  12.141 +    # s is None, so we are done with this request. Signal processor with
  12.142 +    # this so it can take special action if necessary
  12.143 +    processor.finalizeInput(filter)
  12.144 +    
  12.145 +    # we do not remove the processor yet from the registry nor the
  12.146 +    # tramline_id from the request header as this is done in the output phase.
  12.147 +    # The processor can retain state until then.
  12.148 +
  12.149 +    # XXX One problem is that if output does not happen, we do not ever remove
  12.150 +    # the processor, causing a leak.
  12.151 +    
  12.152 +    # close the filter.
  12.153 +    filter.close()
  12.154 +
  12.155 +def outputfilter(filter):
  12.156 +    # we're done if we're in a subrequset
  12.157 +    if filter.req.main is not None:
  12.158 +        pass_on(filter)
  12.159 +        filter.flush()
  12.160 +        return
  12.161 +
  12.162 +    # in case of post request, we may need to do a commit/abort
  12.163 +    # of previous input round
  12.164 +    if filter.req.method == 'POST':
  12.165 +        outputfilter_post(filter)
  12.166 +        return
  12.167 +    # in case of a get request, we may need to serve up files,
  12.168 +    # depending on what's in the headers
  12.169 +    elif filter.req.method == 'GET':
  12.170 +        outputfilter_get(filter)
  12.171 +        return
  12.172 +    
  12.173 +    pass_on(filter)
  12.174 +    filter.flush()
  12.175 +    
  12.176 +def outputfilter_post(filter):
  12.177 +    # get id
  12.178 +    id = filter.req.headers_in.get('tramline_id')
  12.179 +
  12.180 +    if id is None:
  12.181 +        # we're done now, just pass along data
  12.182 +        pass_on(filter)
  12.183 +        filter.flush()
  12.184 +        return
  12.185 +
  12.186 +    # reuse existing processor instance based on id
  12.187 +    processor = theProcessorRegistry.getProcessor(int(id))
  12.188 +
  12.189 +    is_ok = filter.req.headers_out.has_key('tramline_ok')
  12.190 +    if is_ok:
  12.191 +        # if the appserver said okay, 
  12.192 +        processor.commit(filter.req)
  12.193 +    else:
  12.194 +        # if the appserver didn't say okay
  12.195 +        processor.abort()
  12.196 +
  12.197 +    # remove the processor id from the request now, as we're done with it
  12.198 +    theProcessorRegistry.removeProcessor(processor)
  12.199 +    
  12.200 +    # remove the id from the request as well
  12.201 +    del filter.req.headers_in['tramline_id']
  12.202 +
  12.203 +    # now pass along the data.
  12.204 +    pass_on(filter)
  12.205 +    filter.flush()
  12.206 +
  12.207 +def outputfilter_get(filter):
  12.208 +    # check whether we want to do file serving using tramline
  12.209 +    if not filter.req.headers_out.has_key('tramline_file'):
  12.210 +        pass_on(filter)
  12.211 +        filter.flush()
  12.212 +        return
  12.213 +
  12.214 +    status = filter.req.status
  12.215 +
  12.216 +    # now read file id
  12.217 +    data = []
  12.218 +    s = filter.read()
  12.219 +    while s:
  12.220 +        data.append(s)
  12.221 +        s = filter.read()
  12.222 +    file_id = ''.join(data)
  12.223 +    p = id_to_path(tramline_path(filter.req), file_id)
  12.224 +
  12.225 +    log('file id:' + file_id, filter.req)
  12.226 +    # Range request
  12.227 +    hin = filter.req.headers_in
  12.228 +    h = hin.get(TRAMLINE_RANGE_HEADER)
  12.229 +    log('Filter out: Range: ' + str(h), filter.req)
  12.230 +    #TODO log("Filter out: status line " + filter.req.status_line, filter.req)
  12.231 +    log("Filter out: status " + str(status), filter.req)
  12.232 +
  12.233 +    if h and status in [206, 416]:
  12.234 +        # Range was requested and app server agreed
  12.235 +	ranges = parse_range(h)
  12.236 +	if ranges: 
  12.237 +	   serve_ranges(p, ranges[1], filter)
  12.238 +	else:
  12.239 +           log("Unparsable Range header " + h, filter.req)
  12.240 +    elif status == 200:
  12.241 +	serve_file(p, filter)
  12.242 +
  12.243 +def serve_file(p, filter):
  12.244 +    """Serve a whole file."""
  12.245 +    # XXX what if file doesn't exist? 404?
  12.246 +    size = os.stat(p).st_size
  12.247 +    filter.req.headers_out['content-length'] = str(size)
  12.248 +    f = open(p, 'rb')
  12.249 +    dump_file_range(f, 0, size-1, filter)
  12.250 +    f.close()
  12.251 +
  12.252 +def serve_ranges(path, ranges, filter, boundary=None):
  12.253 +    size = os.stat(path).st_size
  12.254 +
  12.255 +    # TODO support suffix notation RFC 2616 p 138
  12.256 +    # do a subroutine for that
  12.257 +
  12.258 +    expanded = [start is None and (size-end, size-1) or 
  12.259 +                (start, end is None and size-1 or end)
  12.260 +                for start, end in ranges]
  12.261 +
  12.262 +    hout = filter.req.headers_out
  12.263 +    fd = open(path, 'rb')
  12.264 +    if len(ranges) == 1:
  12.265 +	start, end = expanded[0]
  12.266 +	if end < start:
  12.267 +             hout['Content-Range'] = 'bytes */%d' % size
  12.268 +             hout['Content-Length'] = str(size)
  12.269 +             filter.req.status = 416
  12.270 +             return
  12.271 +
  12.272 +	log("serve_file_ranges: one chunk, %d-%d" % (start, end), filter.req)
  12.273 +  	hout['Content-Length'] = str(end - start + 1)
  12.274 +        hout['Content-Range'] = 'bytes %d-%d/%d' % (start, end, size)
  12.275 +	dump_file_range(fd, start, end, filter)
  12.276 +	return
  12.277 +
  12.278 +    del hout['Content-Range'] # If present, was from app server and can't be ok
  12.279 +    if boundary is None:
  12.280 +        boundary = mimetools.choose_boundary()
  12.281 +
  12.282 +    content_type = hout['Content-Type']
  12.283 +    # This takes precedence over hout
  12.284 +    filter.req.content_type = 'multipart/byteranges; boundary=%s' % boundary;
  12.285 +
  12.286 +    length = (8 + len(boundary) + # End marker length             
  12.287 +       len(ranges) * (      # Constant lenght per set       
  12.288 +       49 + len(boundary) + len(content_type) + len('%d' % size)))
  12.289 +    for start, end in expanded:
  12.290 +       # Variable length per set                               
  12.291 +       length += len('%d%d' % (start, end)) + 1 + end - start 
  12.292 +    hout['Content-Length'] = str(length)
  12.293 +
  12.294 +    for start, end in expanded:
  12.295 +        filter.write('\r\n--%s\r\n' % boundary)
  12.296 +        filter.write('Content-Type: %s\r\n' % content_type)
  12.297 +        filter.write('Content-Range: bytes %d-%d/%d\r\n\r\n' % (start, end, size))
  12.298 +        dump_file_range(fd, start, end, filter, chunksize=FILE_CHUNKSIZE) # TODO tweak chunksize
  12.299 +    filter.write('\r\n--%s--\r\n' % boundary)
  12.300 +    filter.close()
  12.301 +    fd.close()
  12.302 +
  12.303 +def dump_file_range(fd, start, end, filter, chunksize=FILE_CHUNKSIZE):
  12.304 +    """Perform raw dump of a single file range in filter.
  12.305 +
  12.306 +    Takes chunk size into account."""
  12.307 +    fd.seek(start)
  12.308 +    while end is None or fd.tell() < end+1:
  12.309 +        if end is None:
  12.310 +            data = fd.read(chunksize)
  12.311 +        else:
  12.312 +            data = fd.read(min(chunksize, end+1-fd.tell()))
  12.313 +        if not data:
  12.314 +            break
  12.315 +        filter.write(data)
  12.316 +        # flush the data out as soon as possible, so we don't
  12.317 +        # waste memory
  12.318 +        filter.flush()
  12.319 +
  12.320 +    
  12.321 +class ProcessorRegistry:
  12.322 +    def __init__(self):
  12.323 +        self._processors = {}
  12.324 +
  12.325 +    def getProcessor(self, id):
  12.326 +        return self._processors[id]
  12.327 +
  12.328 +    def createProcessor(self):
  12.329 +        # XXX thread issues?
  12.330 +        while True:
  12.331 +            id = random.randrange(sys.maxint)
  12.332 +            if id not in self._processors:
  12.333 +                break
  12.334 +        result = self._processors[id] = Processor(id)
  12.335 +        return result
  12.336 +
  12.337 +    def removeProcessor(self, processor):
  12.338 +        del self._processors[processor.id]
  12.339 +
  12.340 +theProcessorRegistry = ProcessorRegistry()
  12.341 +
  12.342 +class Processor:
  12.343 +    def __init__(self, id):
  12.344 +        self.id = id
  12.345 +        self._upload_files = []
  12.346 +        self._incoming = []
  12.347 +        # we use a state pattern where the handle method gets
  12.348 +        # replaced by the current handle method for this state.
  12.349 +        self.handle = self.handle_first_boundary
  12.350 +        self.vars_to_handle = []
  12.351 +        self._enable_vars=''
  12.352 +
  12.353 +    def pushInput(self, data, out):
  12.354 +        lines = data.splitlines(True)
  12.355 +        for line in lines:
  12.356 +            self.pushInputLine(line, out)
  12.357 +
  12.358 +    def pushInputLine(self, data, out):
  12.359 +        # collect data
  12.360 +        self._incoming.append(data)
  12.361 +        # if we're not at the end of the line, input was broken
  12.362 +        # somewhere. We return to collect more first.
  12.363 +        if data[-1] != '\n':
  12.364 +            return
  12.365 +        # now use the line in whatever handle method is current
  12.366 +        if len(self._incoming) == 1:
  12.367 +            line = data
  12.368 +        else:
  12.369 +            line = ''.join(self._incoming)
  12.370 +        self._incoming = []
  12.371 +
  12.372 +        self.handle(line, out)
  12.373 +
  12.374 +    def finalizeInput(self, out):
  12.375 +        if self._upload_files:
  12.376 +            out.req.headers_in['tramline'] = ''
  12.377 +
  12.378 +    def commit(self, req):
  12.379 +        # XXX works under the assumption that the last segment of 
  12.380 +        # file path is the tramline id
  12.381 +        for upload_file in self._upload_files:
  12.382 +            dummy, filename = os.path.split(upload_file)
  12.383 +            os.rename(upload_file, id_to_path(tramline_path(req), filename))
  12.384 +
  12.385 +    def abort(self):
  12.386 +        for upload_file in self._upload_files:
  12.387 +            os.remove(upload_file)
  12.388 +    
  12.389 +    def handle_first_boundary(self, line, out):
  12.390 +        self._boundary = line
  12.391 +        self._last_boundary = self._boundary.rstrip() + '--\r\n'
  12.392 +        self.init_headers()
  12.393 +        self.handle = self.handle_headers
  12.394 +        out.write(line)
  12.395 +        
  12.396 +    def init_headers(self):
  12.397 +        self._disposition = None
  12.398 +        self._disposition_options = {}
  12.399 +        self._content_type = 'text/plain'
  12.400 +        self._content_type_options = {}
  12.401 +        
  12.402 +    def handle_headers(self, line, out):
  12.403 +        out.write(line)
  12.404 +        if line in ['\n', '\r\n']:
  12.405 +            self.init_data(out)
  12.406 +            return
  12.407 +        key, value = line.split(':', 1)
  12.408 +        key = key.lower()
  12.409 +        if key == "content-disposition":
  12.410 +            self._disposition, self._disposition_options = parse_header(
  12.411 +                value)
  12.412 +        elif key == "content-type":
  12.413 +            self._content_type, self._content_type_options = parse_header(
  12.414 +                value)
  12.415 +
  12.416 +    def init_data(self, out):
  12.417 +        filename = self._disposition_options.get('filename')
  12.418 +        # if filename is empty, assume no file is submitted and submit
  12.419 +        # empty file -- don't tramline this special case
  12.420 +        if out.req.get_options().get('explicit_enable') and \
  12.421 +              self._disposition_options.get('name')=='tramline_enable':
  12.422 +            self.handle = self.handle_enable_vars
  12.423 +            return
  12.424 +        elif (filename is None or not filename) or \
  12.425 +              out.req.get_options().get('explicit_enable') and \
  12.426 +              self._disposition_options.get('name') not in self.vars_to_handle:
  12.427 +            self.handle = self.handle_data
  12.428 +            return
  12.429 +        fd, pathname, file_id = createUniqueFile(out.req)
  12.430 +
  12.431 +        self._f = os.fdopen(fd, 'wb')
  12.432 +        self._upload_files.append(pathname)
  12.433 +        out.write(file_id)
  12.434 +        out.write('\r\n')
  12.435 +        
  12.436 +        self._previous_line = None
  12.437 +        self.handle = self.handle_file_data
  12.438 +        
  12.439 +    def handle_enable_vars(self, line, out):
  12.440 +        out.write(line)
  12.441 +        if line == self._boundary:
  12.442 +            self.init_headers()
  12.443 +            self.handle = self.handle_headers
  12.444 +            # can be called but once
  12.445 +            # XXX waiting for Emyr's response to write something final - or not
  12.446 +            self.vars_to_handle = self._enable_vars.strip().split()
  12.447 +            self._enable_vars=''
  12.448 +        elif line == self._last_boundary:
  12.449 +            # we should be done
  12.450 +            self.handle = None # shouldn't be called again
  12.451 +        else:
  12.452 +            self._enable_vars+=line
  12.453 +
  12.454 +    def handle_data(self, line, out):
  12.455 +        out.write(line)
  12.456 +        if line == self._boundary:
  12.457 +            self.init_headers()
  12.458 +            self.handle = self.handle_headers
  12.459 +        elif line == self._last_boundary:
  12.460 +            # we should be done
  12.461 +            self.handle = None # shouldn't be called again
  12.462 +
  12.463 +    def handle_file_data(self, line, out):
  12.464 +        if line == self._boundary:
  12.465 +            # write last line, but without \r\n
  12.466 +            self._f.write(self._previous_line[:-2])
  12.467 +            out.write(line)
  12.468 +            self._f.close()
  12.469 +            self._f = None
  12.470 +            self.handle = self.handle_headers
  12.471 +        elif line == self._last_boundary:
  12.472 +            # write last line, but without \r\n
  12.473 +            self._f.write(self._previous_line[:-2])
  12.474 +            out.write(line)
  12.475 +            self._f.close()
  12.476 +            self._f = None
  12.477 +            self.handle = None # shouldn't be called again
  12.478 +        else:
  12.479 +            if self._previous_line is not None:
  12.480 +                self._f.write(self._previous_line)
  12.481 +            self._previous_line = line
  12.482 +
  12.483 +def parse_header(s):
  12.484 +    l = [e.strip() for e in s.split(';')]
  12.485 +    result_value = l.pop(0).lower()
  12.486 +    result_d = {}
  12.487 +    for e in l:
  12.488 +        try:
  12.489 +            key, value = e.split('=', 1)
  12.490 +        except ValueError:
  12.491 +            continue
  12.492 +        key = key.strip().lower()
  12.493 +        value = value.strip()
  12.494 +        if len(value) >= 2 and value.startswith('"') and value.endswith('"'):
  12.495 +            value = value[1:-1]
  12.496 +        result_d[key] = value
  12.497 +    return result_value, result_d
  12.498 +
  12.499 +def createUniqueFile(req):
  12.500 +    """Create a file with unique file id in upload directory.
  12.501 +
  12.502 +    Returns file descriptor, path, like tempfile.mkstemp, but in
  12.503 +    addition returns unique file id.
  12.504 +    """
  12.505 +    create_paths(req)
  12.506 +    # XXX we're relying on implementation of tempfile
  12.507 +    while True:
  12.508 +        file_id = str(random.randrange(sys.maxint))
  12.509 +        # do not accept files already known in the repository
  12.510 +        # this is normally not changing so this should be relatively
  12.511 +        # safe
  12.512 +        if os.path.exists(id_to_path(tramline_path(req), file_id, 
  12.513 +                                     create_intermediate=True)):
  12.514 +            continue # try again
  12.515 +        path = id_to_path(tramline_path(req), file_id, upload=True)
  12.516 +        try:
  12.517 +            fd = os.open(path, tempfile._bin_openflags)
  12.518 +            tempfile._set_cloexec(fd)
  12.519 +            if group_write(req):
  12.520 +               os.chmod(path, 0664)
  12.521 +            return fd, path, file_id
  12.522 +        except OSError, e:
  12.523 +            if e.errno == errno.EEXIST:
  12.524 +                continue # try again
  12.525 +            raise
  12.526 +
  12.527 +def log(data, req):
  12.528 +    f = open(os.path.join(tramline_path(req), 'tramline.log'), 'ab')
  12.529 +    f.write(data)
  12.530 +    f.write('\n')
  12.531 +    f.close()
    13.1 new file mode 100644
    13.2 --- /dev/null
    13.3 +++ b/src/tramline/headers.py
    13.4 @@ -0,0 +1,54 @@
    13.5 +def invalid_byte_range_spec(brs, range, details=None):
    13.6 +   """Raise a ValueError for invalid byte-range-spec.
    13.7 +
    13.8 +   Singled out for code clarity.
    13.9 +   """
   13.10 +   msg = "Invalid byte range spec : '%s' in range header '%s'" % (brs, range)
   13.11 +   if details:
   13.12 +       msg += ' ' + details
   13.13 +   raise ValueError(msg)
   13.14 +
   13.15 +
   13.16 +def parse_range(range):
   13.17 +    """Parse a range header. 
   13.18 +    Adapted from twisted.web2.http_headers.parseHeader. TODO licence issues ?
   13.19 +
   13.20 +    >>> parse_range_header("bytes=1-20")
   13.21 +    ('bytes', (1,20))
   13.22 +    """
   13.23 +
   13.24 +    split = range.strip().split('=')
   13.25 +    if len(split) < 2:
   13.26 +        raise ValueError("Invalid range header format: %s" % range)
   13.27 +
   13.28 +    type = split[0].strip()
   13.29 +    if type != 'bytes':
   13.30 +        raise ValueError("Unknown range unit: %s." % (type,))
   13.31 +    rangeset = (brs.strip() for brs in split[1].split(','))
   13.32 +    ranges = []
   13.33 +
   13.34 +    for byterangespec in rangeset:
   13.35 +        split = [s.strip() for s in byterangespec.split('-')]
   13.36 +        if len(split) != 2 :
   13.37 +            invalid_byte_range_spec(byterangespec, range)
   13.38 +        start, end = split
   13.39 +
   13.40 +        if not start and not end:
   13.41 +            invalid_byte_range_spec(byterangespec, range)
   13.42 +
   13.43 +        if start:
   13.44 +            start = int(start)
   13.45 +        else:
   13.46 +            start = None
   13.47 +
   13.48 +        if end:
   13.49 +            end = int(end)
   13.50 +        else:
   13.51 +            end = None
   13.52 +
   13.53 +        if start and end and start > end:
   13.54 +            invalid_byte_range_spec(byterangespec, range,
   13.55 +                                    details="(start > end)")
   13.56 +
   13.57 +        ranges.append((start,end))
   13.58 +    return type,ranges
    14.1 new file mode 100644
    14.2 --- /dev/null
    14.3 +++ b/src/tramline/interfaces.py
    14.4 @@ -0,0 +1,37 @@
    14.5 +class IInputFilterProcessor:
    14.6 +    def pushInput(data, out):
    14.7 +        """Push block of inputted data into processor.
    14.8 +
    14.9 +        Processor writes data to pass along on input stream to
   14.10 +        out, using .write().
   14.11 +        """
   14.12 +
   14.13 +    def finalizeInput(out):
   14.14 +        """Notify processor that input data is now complete.
   14.15 +
   14.16 +        Processor can still choose to write data to pass along input
   14.17 +        stream to out, using .write().
   14.18 +        """
   14.19 +
   14.20 +    def commit():
   14.21 +        """Commit any action taken during the input phase.
   14.22 +        """
   14.23 +
   14.24 +    def abort():
   14.25 +        """"Abort action taken in the input phase.
   14.26 +        """
   14.27 +                
   14.28 +class IOutputFilterProcessor:
   14.29 +    def pushOutput(data, out):
   14.30 +        """Push block of outputted data into processor.
   14.31 +
   14.32 +        Processor writes data to pass along on output stream to
   14.33 +        out, using .write().
   14.34 +        """
   14.35 +
   14.36 +    def finalizeOutput(self, out):
   14.37 +        """Notify processor that output data is now complete.
   14.38 +
   14.39 +        Processor can still choose to write data to pass along output
   14.40 +        stream to out, using .write().
   14.41 +        """
    15.1 new file mode 100644
    15.2 --- /dev/null
    15.3 +++ b/src/tramline/simplefilter.py
    15.4 @@ -0,0 +1,45 @@
    15.5 +#from mod_python import apache, util
    15.6 +import os, shutil
    15.7 +
    15.8 +def log(text):
    15.9 +    f = open('/tmp/tramline.log', 'a')
   15.10 +    f.write(text)
   15.11 +    f.write('\n')
   15.12 +    f.close()
   15.13 +    
   15.14 +def inputfilter(filter):
   15.15 +    if filter.req.method != 'POST':
   15.16 +        filter.disable()
   15.17 +        return
   15.18 +    #del filter.req.headers_in['Content-Length']
   15.19 +    f = open('/tmp/filtertest.txt', 'ab')
   15.20 +    log('first read')
   15.21 +    s = filter.read()
   15.22 +    while s:
   15.23 +        #s = s[:200] + s[205:]
   15.24 +        log('writing (%s)' % len(s))
   15.25 +        # if 'head' not in s:
   15.26 +        f.write(s)
   15.27 +        f.flush()
   15.28 +        filter.write(s)
   15.29 +        log('loop read')
   15.30 +        s = filter.read()
   15.31 +    if s is None:
   15.32 +        log('closing')
   15.33 +        #filter.flush()
   15.34 +        filter.close()
   15.35 +        raise "error"
   15.36 +        #filter.disable()
   15.37 +    f.close()
   15.38 +
   15.39 +def requesthandler(req):
   15.40 +    
   15.41 +    fs = util.FieldStorage(req)
   15.42 +    for key in fs.keys():
   15.43 +        value = fs[key]
   15.44 +        if isinstance(value, util.Field):
   15.45 +            f = open(os.path.join('/tmp/dumpingground', value.filename), 'wb')
   15.46 +            shutil.copyfileobj(value.file, f)
   15.47 +            f.close()
   15.48 +            
   15.49 +    return apache.DECLINED
    16.1 new file mode 100644
    16.2 --- /dev/null
    16.3 +++ b/src/tramline/tests/__init__.py
    16.4 @@ -0,0 +1,1 @@
    16.5 +# this is a package
    17.1 new file mode 100644
    17.2 --- /dev/null
    17.3 +++ b/src/tramline/tests/data/input1.txt
    17.4 @@ -0,0 +1,11 @@
    17.5 +-----------------------------100323068321119442571506749230
    17.6 +Content-Disposition: form-data; name="test"
    17.7 +Content-Type: application/octet-stream
    17.8 +
    17.9 +first line
   17.10 +
   17.11 +-----------------------------100323068321119442571506749230
   17.12 +Content-Disposition: form-data; name="submit"
   17.13 +
   17.14 +submit data
   17.15 +-----------------------------100323068321119442571506749230--
    18.1 new file mode 100644
    18.2 --- /dev/null
    18.3 +++ b/src/tramline/tests/data/input2.txt
    18.4 @@ -0,0 +1,12 @@
    18.5 +-----------------------------100323068321119442571506749230
    18.6 +Content-Disposition: form-data; filename="test.txt"; name="test"
    18.7 +Content-Type: application/octet-stream
    18.8 +
    18.9 +first line
   18.10 +second line
   18.11 +
   18.12 +-----------------------------100323068321119442571506749230
   18.13 +Content-Disposition: form-data; name="submit"
   18.14 +
   18.15 +submit data
   18.16 +-----------------------------100323068321119442571506749230--
    19.1 new file mode 100644
    19.2 --- /dev/null
    19.3 +++ b/src/tramline/tests/data/input3.txt
    19.4 @@ -0,0 +1,11 @@
    19.5 +-----------------------------100323068321119442571506749230
    19.6 +Content-Disposition: form-data; name="test"
    19.7 +Content-Type: application/octet-stream
    19.8 +
    19.9 +first line
   19.10 +
   19.11 +-----------------------------100323068321119442571506749230
   19.12 +Content-Disposition: form-data; name="submit"
   19.13 +
   19.14 +submit data
   19.15 +-----------------------------100323068321119442571506749230--
    20.1 new file mode 100644
    20.2 --- /dev/null
    20.3 +++ b/src/tramline/tests/data/input4.txt
    20.4 @@ -0,0 +1,11 @@
    20.5 +-----------------------------100323068321119442571506749230
    20.6 +Content-Disposition: form-data; filename="test.txt"; name="test"
    20.7 +Content-Type: application/octet-stream
    20.8 +
    20.9 +first line
   20.10 +second line
   20.11 +-----------------------------100323068321119442571506749230
   20.12 +Content-Disposition: form-data; name="submit"
   20.13 +
   20.14 +submit data
   20.15 +-----------------------------100323068321119442571506749230--
    21.1 new file mode 100644
    21.2 --- /dev/null
    21.3 +++ b/src/tramline/tests/data/input5.txt
    21.4 @@ -0,0 +1,10 @@
    21.5 +-----------------------------100323068321119442571506749230
    21.6 +Content-Disposition: form-data; name="form.pdf"; filename=""
    21.7 +Content-Type: application/octet-stream
    21.8 +
    21.9 +
   21.10 +-----------------------------100323068321119442571506749230
   21.11 +Content-Disposition: form-data; name="submit"
   21.12 +
   21.13 +submit data
   21.14 +-----------------------------100323068321119442571506749230--
    22.1 new file mode 100644
    22.2 --- /dev/null
    22.3 +++ b/src/tramline/tests/data/input6.txt
    22.4 @@ -0,0 +1,17 @@
    22.5 +-----------------------------100323068321119442571506749230
    22.6 +Content-Disposition: form-data; name="tramline_enable"
    22.7 +
    22.8 +yorg test
    22.9 +other
   22.10 +-----------------------------100323068321119442571506749230
   22.11 +Content-Disposition: form-data; filename="test.txt"; name="test"
   22.12 +Content-Type: application/octet-stream
   22.13 +
   22.14 +first line
   22.15 +second line
   22.16 +
   22.17 +-----------------------------100323068321119442571506749230
   22.18 +Content-Disposition: form-data; name="submit"
   22.19 +
   22.20 +submit data
   22.21 +-----------------------------100323068321119442571506749230--
    23.1 new file mode 100755
    23.2 --- /dev/null
    23.3 +++ b/src/tramline/tests/data/input7.txt
    23.4 @@ -0,0 +1,10 @@
    23.5 +-----------------------------100323068321119442571506749230
    23.6 +Content-Disposition: form-data; filename="test.txt"; name="test"
    23.7 +Content-Type: application/octet-stream
    23.8 +
    23.9 +abcdefghijklmnopqrstuvwxyz
   23.10 +-----------------------------100323068321119442571506749230
   23.11 +Content-Disposition: form-data; name="submit"
   23.12 +
   23.13 +submit data
   23.14 +-----------------------------100323068321119442571506749230--
    24.1 new file mode 100755
    24.2 --- /dev/null
    24.3 +++ b/src/tramline/tests/data/output7_ranges.txt
    24.4 @@ -0,0 +1,17 @@
    24.5 +
    24.6 +--test-boundary
    24.7 +Content-Type: application/tramtest
    24.8 +Content-Range: bytes 2-7/26
    24.9 +
   24.10 +cdefgh
   24.11 +--test-boundary
   24.12 +Content-Type: application/tramtest
   24.13 +Content-Range: bytes 21-25/26
   24.14 +
   24.15 +vwxyz
   24.16 +--test-boundary
   24.17 +Content-Type: application/tramtest
   24.18 +Content-Range: bytes 1-1/26
   24.19 +
   24.20 +b
   24.21 +--test-boundary--
    25.1 new file mode 100644
    25.2 --- /dev/null
    25.3 +++ b/src/tramline/tests/test_core.py
    25.4 @@ -0,0 +1,542 @@
    25.5 +import os, shutil, sys, stat
    25.6 +import unittest
    25.7 +from StringIO import StringIO
    25.8 +
    25.9 +from tramline.core import inputfilter, outputfilter, tramline_upload_path,\
   25.10 +     tramline_repository_path, parse_header, create_paths, tramline_path,\
   25.11 +     serve_ranges, id_to_path
   25.12 +
   25.13 +from tramline.core import OPTION_ALLOW_GROUP_WRITE
   25.14 +from tramline.core import TRAMLINE_RANGE_HEADER
   25.15 +
   25.16 +tramline_path = '/tmp/trampath'
   25.17 +
   25.18 +class StringTable(dict):
   25.19 +    def __setitem__(self, key, value):
   25.20 +        if not isinstance(value, str):
   25.21 +            raise ValueError("Table values must be strings")
   25.22 +        dict.__setitem__(self, key, value)
   25.23 +
   25.24 +class Request:
   25.25 +    def __init__(self, method):
   25.26 +        self.headers_in = StringTable({'Content-Type' : 'multipart/form-data'})
   25.27 +        self.headers_out = StringTable()
   25.28 +        self.main = None
   25.29 +        self.method = method
   25.30 +        self.options = {'tramline_path': tramline_path}
   25.31 +
   25.32 +    def get_options(self):
   25.33 +        return self.options
   25.34 +        
   25.35 +class Filter:
   25.36 +    def __init__(self, input, output, is_last=True, method='POST'):
   25.37 +        self.input = input
   25.38 +        self.output = output
   25.39 +        self.is_closed = False
   25.40 +        self.req = Request(method)
   25.41 +        self.is_last = is_last
   25.42 +        
   25.43 +    def read(self, length=None):
   25.44 +        data = self.input.read()
   25.45 +        if data == '' and self.is_last:
   25.46 +            return None
   25.47 +        return data
   25.48 +        
   25.49 +    def readline(self, length=None):
   25.50 +        data = self.input.readline()
   25.51 +        if data == '' and self.is_last:
   25.52 +            return None
   25.53 +        return data
   25.54 +    
   25.55 +    def write(self, data):
   25.56 +        self.output.write(data)
   25.57 +        
   25.58 +    def close(self):
   25.59 +        self.is_closed = True
   25.60 +
   25.61 +    def disable(self):
   25.62 +        pass
   25.63 +
   25.64 +    def pass_on(self):
   25.65 +        data = self.input.read()
   25.66 +        self.output.write(data)
   25.67 +
   25.68 +    disable = pass_on
   25.69 +
   25.70 +    def flush(self):
   25.71 +        pass
   25.72 +    
   25.73 +class TramlineTests(unittest.TestCase):
   25.74 +
   25.75 +    def setUp(self):
   25.76 +        pass
   25.77 +# XXX is cleaning up the right way to go? it's hard to debug test failures
   25.78 +# then and previous real uploads are gone..
   25.79 +#        shutil.rmtree(tramline_path, ignore_errors=True)
   25.80 +#        create_paths()
   25.81 +        
   25.82 +    def tearDown(self):
   25.83 +        pass
   25.84 +#        shutil.rmtree(tramline_path, ignore_errors=True)
   25.85 +    
   25.86 +    def test_inputfilter(self):
   25.87 +        input = open(get_data_path('input1.txt'), 'rb')
   25.88 +        output = StringIO()
   25.89 +        filter = Filter(input, output)
   25.90 +        
   25.91 +        inputfilter(filter)
   25.92 +
   25.93 +        input.close()
   25.94 +
   25.95 +        f = open(get_data_path('input1.txt'), 'rb')
   25.96 +        data = f.read()
   25.97 +        f.close()
   25.98 +
   25.99 +        output_data = output.getvalue()
  25.100 +        self.assertEquals(data, output_data)
  25.101 +        self.assert_(filter.is_closed)
  25.102 +
  25.103 +        self.assert_('tramline' not in filter.req.headers_in)
  25.104 +        
  25.105 +    def test_inputfilter_file(self):
  25.106 +        input = open(get_data_path('input2.txt'), 'rb')
  25.107 +        output = StringIO()
  25.108 +        filter = Filter(input, output)
  25.109 +        
  25.110 +        inputfilter(filter)
  25.111 +
  25.112 +        input.close()
  25.113 +
  25.114 +        output_data = output.getvalue()
  25.115 +
  25.116 +        file_id = self.file_id(output_data)
  25.117 +        f = open(os.path.join(tramline_upload_path(filter.req), file_id), 'rb')
  25.118 +        data = f.read()
  25.119 +        f.close()
  25.120 +        self.assertEquals(
  25.121 +            'first line\nsecond line\n', data)
  25.122 +        self.assert_('tramline' in filter.req.headers_in)
  25.123 +        
  25.124 +    def test_inputfilter_file2(self):
  25.125 +        input = open(get_data_path('input4.txt'), 'rb')
  25.126 +        output = StringIO()
  25.127 +        filter = Filter(input, output)
  25.128 +        
  25.129 +        inputfilter(filter)
  25.130 +
  25.131 +        input.close()
  25.132 +
  25.133 +        output_data = output.getvalue()
  25.134 +
  25.135 +        file_id = self.file_id(output_data)
  25.136 +        f = open(os.path.join(tramline_upload_path(filter.req), file_id), 'rb')
  25.137 +        
  25.138 +        data = f.read()
  25.139 +        f.close()
  25.140 +        self.assertEquals(
  25.141 +            'first line\nsecond line', data)
  25.142 +        
  25.143 +    def test_split_filter(self):
  25.144 +        f = open(get_data_path('input2.txt'), 'rb')
  25.145 +        data = f.read()
  25.146 +        f.close()
  25.147 +        halfway = len(data) / 2
  25.148 +        first_half = data[:halfway]
  25.149 +        second_half = data[halfway:]
  25.150 +
  25.151 +        output = StringIO()
  25.152 +        
  25.153 +        filter = Filter(StringIO(first_half), output, is_last=False)
  25.154 +        inputfilter(filter)
  25.155 +        filter.input = StringIO(second_half)
  25.156 +        filter.is_last = True
  25.157 +        inputfilter(filter)
  25.158 +        
  25.159 +        output_data = output.getvalue()
  25.160 +
  25.161 +        file_id = self.file_id(output_data)
  25.162 +        f = open(os.path.join(tramline_upload_path(filter.req), file_id), 'rb')
  25.163 +
  25.164 +        data = f.read()
  25.165 +        f.close()
  25.166 +        self.assertEquals(
  25.167 +            'first line\nsecond line\n', data)
  25.168 +
  25.169 +    def test_three_split_filter(self):
  25.170 +        f = open(get_data_path('input2.txt'), 'rb')
  25.171 +        data = f.read()
  25.172 +        f.close()
  25.173 +
  25.174 +        third = len(data) / 3
  25.175 +        first = data[:third]
  25.176 +        second = data[third:third + third]
  25.177 +        third = data[third + third:]
  25.178 +        
  25.179 +        output = StringIO()
  25.180 +        
  25.181 +        filter = Filter(StringIO(first), output, is_last=False)
  25.182 +        inputfilter(filter)
  25.183 +        filter.input = StringIO(second)
  25.184 +        inputfilter(filter)
  25.185 +        filter.input = StringIO(third)
  25.186 +        filter.is_last = True
  25.187 +        inputfilter(filter)
  25.188 +        
  25.189 +        output_data = output.getvalue()
  25.190 +
  25.191 +        file_id = self.file_id(output_data)
  25.192 +        f = open(os.path.join(tramline_upload_path(filter.req), file_id), 'rb')
  25.193 +        data = f.read()
  25.194 +        f.close()
  25.195 +        self.assertEquals(
  25.196 +            'first line\nsecond line\n', data)
  25.197 +
  25.198 +    def test_empty_file(self):
  25.199 +        input = open(get_data_path('input5.txt'), 'rb')
  25.200 +        output = StringIO()
  25.201 +        filter = Filter(input, output)
  25.202 +
  25.203 +        inputfilter(filter)
  25.204 +
  25.205 +        input.close()
  25.206 +
  25.207 +        output_data = output.getvalue()
  25.208 +
  25.209 +        file_id = self.file_id(output_data)
  25.210 +        self.assertEquals('', file_id)
  25.211 +        
  25.212 +    def test_abort(self):
  25.213 +        input = open(get_data_path('input2.txt'), 'rb')
  25.214 +        output = StringIO()
  25.215 +        filter = Filter(input, output)
  25.216 +
  25.217 +        inputfilter(filter)
  25.218 +        input.close()
  25.219 +
  25.220 +        output_data = output.getvalue()
  25.221 +
  25.222 +        file_id = self.file_id(output_data)
  25.223 +
  25.224 +        tramline_id = filter.req.headers_in['tramline_id']
  25.225 +        
  25.226 +        # now send 'foo' back to the client in the response
  25.227 +        output = StringIO()
  25.228 +        filter = Filter(StringIO('foo'), output)
  25.229 +        filter.req.headers_in['tramline_id'] = tramline_id
  25.230 +        # don't send 'tramline_ok'
  25.231 +        
  25.232 +        # now send output
  25.233 +        outputfilter(filter)
  25.234 +
  25.235 +        # we should get 'foo'
  25.236 +        self.assertEquals('foo', output.getvalue())
  25.237 +        
  25.238 +        # file should be gone
  25.239 +        self.assert_(not os.path.exists(os.path.join(
  25.240 +            tramline_upload_path(filter.req), file_id)))
  25.241 +        # and not be stored
  25.242 +        self.assert_(not os.path.exists(os.path.join(
  25.243 +            tramline_repository_path(filter.req), file_id)))
  25.244 +
  25.245 +    def test_commit(self):
  25.246 +        input = open(get_data_path('input2.txt'), 'rb')
  25.247 +        output = StringIO()
  25.248 +        filter = Filter(input, output)
  25.249 +
  25.250 +        inputfilter(filter)
  25.251 +        input.close()
  25.252 +
  25.253 +        output_data = output.getvalue()
  25.254 +
  25.255 +        file_id = self.file_id(output_data)
  25.256 +
  25.257 +        tramline_id = filter.req.headers_in['tramline_id']
  25.258 +        
  25.259 +        # now send 'foo' back to the client in the response
  25.260 +        output = StringIO()
  25.261 +        filter = Filter(StringIO('foo'), output)
  25.262 +        filter.req.headers_in['tramline_id'] = tramline_id
  25.263 +        # send tramline_ok
  25.264 +        filter.req.headers_out['tramline_ok'] = 'something'
  25.265 +
  25.266 +        # now send output
  25.267 +        outputfilter(filter)
  25.268 +
  25.269 +        # we should get 'foo'
  25.270 +        self.assertEquals('foo', output.getvalue())
  25.271 +        
  25.272 +        # file should be gone in upload
  25.273 +        self.assert_(not os.path.exists(os.path.join(
  25.274 +            tramline_upload_path(filter.req), file_id)))
  25.275 +        # and should be stored
  25.276 +        self.assert_(os.path.exists(os.path.join(
  25.277 +            tramline_repository_path(filter.req), file_id)))
  25.278 +
  25.279 +    def test_group_perms(self):
  25.280 +        if sys.platform == 'win32':
  25.281 +            return
  25.282 +
  25.283 +        # remove previous runs
  25.284 +        shutil.rmtree(tramline_path)
  25.285 +
  25.286 +        # upload simulation: input
  25.287 +        input = open(get_data_path('input2.txt'), 'rb')
  25.288 +        output = StringIO()
  25.289 +        filter = Filter(input, output)
  25.290 +        filter.req.options[OPTION_ALLOW_GROUP_WRITE] = 'True'
  25.291 +        inputfilter(filter)
  25.292 +        input.close()
  25.293 +
  25.294 +        output_data = output.getvalue()
  25.295 +
  25.296 +        file_id = self.file_id(output_data)
  25.297 +
  25.298 +        tramline_id = filter.req.headers_in['tramline_id']
  25.299 +
  25.300 +        # now the output filter part
  25.301 +        output = StringIO()
  25.302 +        filter = Filter(StringIO('foo'), output)
  25.303 +        filter.req.options[OPTION_ALLOW_GROUP_WRITE] = 'True'
  25.304 +        filter.req.headers_in['tramline_id'] = tramline_id
  25.305 +        filter.req.headers_out['tramline_ok'] = 'something'
  25.306 +        outputfilter(filter)
  25.307 +
  25.308 +        # assertions: file got stored with right permissions
  25.309 +        path = os.path.join(tramline_repository_path(filter.req), file_id)
  25.310 +        self.assert_(os.path.exists(path))
  25.311 +        self.assertEquals(0664, stat.S_IMODE(os.stat(path).st_mode))
  25.312 +
  25.313 +    def test_get_output(self):
  25.314 +        # send 'foo' back to the client
  25.315 +        output = StringIO()
  25.316 +        filter = Filter(StringIO('foo'), output, method='GET')
  25.317 +        # set no special tramline headers
  25.318 +        # now send output
  25.319 +        outputfilter(filter)
  25.320 +        # we should get 'foo'
  25.321 +        self.assertEquals('foo', output.getvalue())
  25.322 +
  25.323 +    def test_file_serve(self):
  25.324 +        # first upload file and commit
  25.325 +        input = open(get_data_path('input2.txt'), 'rb')
  25.326 +        output = StringIO()
  25.327 +        filter = Filter(input, output)
  25.328 +
  25.329 +        inputfilter(filter)
  25.330 +        input.close()
  25.331 +
  25.332 +        output_data = output.getvalue()
  25.333 +
  25.334 +        file_id = self.file_id(output_data)
  25.335 +
  25.336 +        tramline_id = filter.req.headers_in['tramline_id']
  25.337 +        
  25.338 +        # now send 'foo' back to the client in the response
  25.339 +        output = StringIO()
  25.340 +        filter = Filter(StringIO('foo'), output)
  25.341 +        filter.req.headers_in['tramline_id'] = tramline_id
  25.342 +        # send tramline_ok
  25.343 +        filter.req.headers_out['tramline_ok'] = 'something'
  25.344 +
  25.345 +        # now send output
  25.346 +        outputfilter(filter)
  25.347 +
  25.348 +        # now send file as output
  25.349 +        
  25.350 +        output = StringIO()
  25.351 +        # send file_id in response body
  25.352 +        filter = Filter(StringIO(file_id), output, method='GET')
  25.353 +        # set tramline_file header to notify tramline to take action
  25.354 +        filter.req.headers_out['tramline_file'] = ''
  25.355 +        filter.req.status = 200
  25.356 +        # now send
  25.357 +        outputfilter(filter)
  25.358 +        # output should now contain file
  25.359 +        f = open(os.path.join(
  25.360 +            tramline_repository_path(filter.req), file_id), 'rb')
  25.361 +        expected_data = f.read()
  25.362 +        f.close()
  25.363 +        data = output.getvalue()
  25.364 +        self.assertEquals(expected_data, data)
  25.365 +        
  25.366 +    def test_range_serve(self):
  25.367 +        # first upload file and commit
  25.368 +        input = open(get_data_path('input7.txt'), 'rb')
  25.369 +        output = StringIO()
  25.370 +        filter = Filter(input, output)
  25.371 +
  25.372 +        inputfilter(filter)
  25.373 +        input.close()
  25.374 +
  25.375 +        output_data = output.getvalue()
  25.376 +
  25.377 +        file_id = self.file_id(output_data)
  25.378 +
  25.379 +        tramline_id = filter.req.headers_in['tramline_id']
  25.380 +        
  25.381 +        # now send 'foo' back to the client in the response
  25.382 +        output = StringIO()
  25.383 +        filter = Filter(StringIO('foo'), output)
  25.384 +        filter.req.headers_in['tramline_id'] = tramline_id
  25.385 +        # send tramline_ok
  25.386 +        filter.req.headers_out['tramline_ok'] = 'something'
  25.387 +
  25.388 +        # now send output
  25.389 +        outputfilter(filter)
  25.390 +
  25.391 +        # simple byte-range-spec
  25.392 +        output = StringIO()
  25.393 +        filter = Filter(StringIO(file_id), output, method='GET')
  25.394 +        filter.req.headers_out['tramline_file'] = ''
  25.395 +        filter.req.status = 206
  25.396 +        filter.req.headers_in[TRAMLINE_RANGE_HEADER] = 'bytes = 1-5'
  25.397 +        outputfilter(filter)
  25.398 +        self.assertEquals('bcdef', output.getvalue())
  25.399 +        self.assertEquals('5', filter.req.headers_out['Content-Length'])
  25.400 +
  25.401 +        # borderline simple byte-range-spec
  25.402 +        output = StringIO()
  25.403 +        filter = Filter(StringIO(file_id), output, method='GET')
  25.404 +        filter.req.headers_out['tramline_file'] = ''
  25.405 +        filter.req.status = 206
  25.406 +        filter.req.headers_in[TRAMLINE_RANGE_HEADER] = 'bytes = 0-0'
  25.407 +        outputfilter(filter)
  25.408 +        self.assertEquals('a', output.getvalue())
  25.409 +        self.assertEquals('1', filter.req.headers_out['Content-Length'])
  25.410 +
  25.411 +        # till the end byte-range-spec
  25.412 +        output = StringIO()
  25.413 +        filter = Filter(StringIO(file_id), output, method='GET')
  25.414 +        filter.req.headers_out['tramline_file'] = ''
  25.415 +        filter.req.status = 206
  25.416 +        filter.req.headers_in[TRAMLINE_RANGE_HEADER] = 'bytes = 15-'
  25.417 +        outputfilter(filter)
  25.418 +        self.assertEquals('pqrstuvwxyz', output.getvalue())
  25.419 +
  25.420 +        # suffix-byte-range-spec
  25.421 +        output = StringIO()
  25.422 +        filter = Filter(StringIO(file_id), output, method='GET')
  25.423 +        filter.req.headers_out['tramline_file'] = ''
  25.424 +        filter.req.status = 206
  25.425 +        filter.req.headers_in[TRAMLINE_RANGE_HEADER] = 'bytes = -5'
  25.426 +        outputfilter(filter)
  25.427 +        self.assertEquals('vwxyz', output.getvalue())
  25.428 +        self.assertEquals('5', filter.req.headers_out['Content-Length'])
  25.429 +
  25.430 +        # several ranges
  25.431 +        output = StringIO()
  25.432 +        filter = Filter(StringIO(file_id), output, method='GET')
  25.433 +        filter.req.headers_out['tramline_file'] = ''
  25.434 +        filter.req.headers_out['Content-Type'] = 'application/tramtest'
  25.435 +        filter.req.headers_out['Content-Range'] = '0-9/10'
  25.436 +        filter.req.status = 206
  25.437 +        range = 'bytes = 2-7,-5,1-1'
  25.438 +        from tramline.headers import parse_range
  25.439 +	ranges = parse_range(range)[1]
  25.440 +        serve_ranges(id_to_path(tramline_path, file_id),
  25.441 +                     ranges, filter,
  25.442 +                     boundary="test-boundary")
  25.443 +
  25.444 +        expf = open(get_data_path('output7_ranges.txt'), 'r')
  25.445 +        expected = expf.read()
  25.446 +        expf.close()
  25.447 +        produced = output.getvalue()
  25.448 +        self.assertEquals(expected, produced)
  25.449 +        self.assertEquals(str(len(produced)), 
  25.450 +                          filter.req.headers_out['Content-Length'])
  25.451 +        
  25.452 +    def test_parse_header(self):
  25.453 +        name, d = parse_header('form-data; name="test"')
  25.454 +        self.assertEquals('form-data',
  25.455 +                          name)
  25.456 +        self.assertEquals('test',
  25.457 +                          d['name'])
  25.458 +        
  25.459 +    def test_parse_header_nothing(self):
  25.460 +        name, d = parse_header('form-data')
  25.461 +        self.assertEquals('form-data',
  25.462 +                          name)
  25.463 +        self.assertEquals(0, len(d))
  25.464 +
  25.465 +    def test_parse_header_multiple(self):
  25.466 +        name, d = parse_header('form-data; name="test"; filename="foo"')
  25.467 +        self.assertEquals('form-data',
  25.468 +                          name)
  25.469 +        self.assertEquals(2, len(d))
  25.470 +        self.assertEquals('test',
  25.471 +                          d['name'])
  25.472 +        self.assertEquals('foo',
  25.473 +                          d['filename'])
  25.474 +
  25.475 +    def test_parse_header_whitespace(self):
  25.476 +        name, d = parse_header('form-data;  name="test";filename="foo" ')
  25.477 +        self.assertEquals('form-data',
  25.478 +                          name)
  25.479 +        self.assertEquals(2, len(d))
  25.480 +        self.assertEquals('test',
  25.481 +                          d['name'])
  25.482 +        self.assertEquals('foo',
  25.483 +                          d['filename'])
  25.484 +
  25.485 +    def test_parse_header_noquotes(self):
  25.486 +        name, d = parse_header('form-data; name=test;filename=foo')
  25.487 +        self.assertEquals('form-data',
  25.488 +                          name)
  25.489 +        self.assertEquals(2, len(d))
  25.490 +        self.assertEquals('test',
  25.491 +                          d['name'])
  25.492 +        self.assertEquals('foo',
  25.493 +                          d['filename'])
  25.494 +
  25.495 +    def test_explicit_enabling(self):
  25.496 +        # first make sure we fail in normal mode when explicit
  25.497 +        # enabling is enabled (boy this term needs something better)
  25.498 +        input = open(get_data_path('input2.txt'), 'rb')
  25.499 +        output = StringIO()
  25.500 +        filter = Filter(input, output)
  25.501 +        filter.req.options['explicit_enable'] = True
  25.502 +
  25.503 +        inputfilter(filter)
  25.504 +
  25.505 +        input.close()
  25.506 +
  25.507 +        output_data = output.getvalue()
  25.508 +
  25.509 +        file_id = self.file_id(output_data)
  25.510 +        self.assert_(not os.path.isfile(os.path.join(tramline_upload_path(filter.req), file_id)))
  25.511 +
  25.512 +        # now go at it again and make sure it works this time
  25.513 +        input = open(get_data_path('input6.txt'), 'rb')
  25.514 +        output = StringIO()
  25.515 +        filter = Filter(input, output)
  25.516 +        filter.req.options['explicit_enable'] = True
  25.517 +
  25.518 +        inputfilter(filter)
  25.519 +
  25.520 +        input.close()
  25.521 +
  25.522 +        output_data = output.getvalue()
  25.523 +
  25.524 +        file_id = self.file_id(output_data)
  25.525 +        self.assert_(os.path.isfile(os.path.join(tramline_upload_path(filter.req), file_id)))
  25.526 +
  25.527 +    def file_id(self, data, start=0):
  25.528 +        i = data.find('filename', start)
  25.529 +        if i == -1:
  25.530 +            return None
  25.531 +        i = data.find('\r\n\r\n', i) + 4
  25.532 +        j = data.find('\r\n', i)
  25.533 +        id = data[i:j].strip()
  25.534 +        return id
  25.535 +    
  25.536 +def get_data_path(name):
  25.537 +    path, rest = os.path.split(__file__)
  25.538 +    return os.path.join(path, 'data', name)
  25.539 +    
  25.540 +def test_suite():
  25.541 +    suite = unittest.TestSuite()
  25.542 +    suite.addTest(unittest.makeSuite(TramlineTests))
  25.543 +    return suite
  25.544 +
  25.545 +if __name__ == '__main__':
  25.546 +    unittest.main()