github.com/olivere/camlistore@v0.0.0-20140121221811-1b7ac2da0199/server/gae-py-blobserver/main.py (about)

     1  #!/usr/bin/env python
     2  #
     3  # Camlistore blob server for App Engine.
     4  #
     5  # Derived from Brad's Brackup-gae utility:
     6  #   http://github.com/bradfitz/brackup-gae-server
     7  #
     8  # Copyright 2010 Google Inc.
     9  #
    10  # Licensed under the Apache License, Version 2.0 (the "License");
    11  # you may not use this file except in compliance with the License.
    12  # You may obtain a copy of the License at
    13  #
    14  #     http://www.apache.org/licenses/LICENSE-2.0
    15  #
    16  # Unless required by applicable law or agreed to in writing, software
    17  # distributed under the License is distributed on an "AS IS" BASIS,
    18  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    19  # See the License for the specific language governing permissions and
    20  # limitations under the License.
    21  #
    22  
    23  """Upload server for camlistore.
    24  
    25  To test:
    26  
    27  # Stat -- 200 response
    28  curl -v \
    29    -d camliversion=1 \
    30    http://localhost:8080/camli/stat
    31  
    32  # Upload -- 200 response
    33  curl -v -L \
    34    -F sha1-126249fd8c18cbb5312a5705746a2af87fba9538=@./test_data.txt \
    35    <the url returned by stat>
    36  
    37  # Put with bad blob_ref parameter -- 400 response
    38  curl -v -L \
    39    -F sha1-22a7fdd575f4c3e7caa3a55cc83db8b8a6714f0f=@./test_data.txt \
    40    <the url returned by stat>
    41  
    42  # Get present -- the blob
    43  curl -v http://localhost:8080/camli/\
    44  sha1-126249fd8c18cbb5312a5705746a2af87fba9538
    45  
    46  # Get missing -- 404
    47  curl -v http://localhost:8080/camli/\
    48  sha1-22a7fdd575f4c3e7caa3a55cc83db8b8a6714f0f
    49  
    50  # Check present -- 200 with only headers
    51  curl -I http://localhost:8080/camli/\
    52  sha1-126249fd8c18cbb5312a5705746a2af87fba9538
    53  
    54  # Check missing -- 404 with empty list response
    55  curl -I http://localhost:8080/camli/\
    56  sha1-22a7fdd575f4c3e7caa3a55cc83db8b8a6714f0f
    57  
    58  # List -- 200 with list of blobs (just one)
    59  curl -v http://localhost:8080/camli/enumerate-blobs&limit=1
    60  
    61  # List offset -- 200 with list of no blobs
    62  curl -v http://localhost:8080/camli/enumerate-blobs?after=\
    63  sha1-126249fd8c18cbb5312a5705746a2af87fba9538
    64  
    65  """
    66  
    67  import cgi
    68  import hashlib
    69  import logging
    70  import urllib
    71  import wsgiref.handlers
    72  
    73  from google.appengine.ext import blobstore
    74  from google.appengine.ext import db
    75  from google.appengine.ext import webapp
    76  from google.appengine.ext.webapp import blobstore_handlers
    77  
    78  import config
    79  
    80  
    81  class Blob(db.Model):
    82    """Some content-addressable blob.
    83  
    84    The key is the algorithm, dash, and the lowercase hex digest:
    85      "sha1-f1d2d2f924e986ac86fdf7b36c94bcdf32beec15"
    86    """
    87  
    88    # The actual bytes.
    89    blob = blobstore.BlobReferenceProperty(indexed=False)
    90  
    91    # Size.  (already in the blobinfo, but denormalized for speed)
    92    size = db.IntegerProperty(indexed=False)
    93  
    94  
    95  class HelloHandler(webapp.RequestHandler):
    96    """Present ourselves to the world."""
    97  
    98    def get(self):
    99      self.response.out.write('Hello!  This is an AppEngine Camlistore '
   100                              'blob server.<p>')
   101      self.response.out.write('<a href=js/index.html>js frontend</a>')
   102  
   103  
   104  class ListHandler(webapp.RequestHandler):
   105    """Return chunks that the server has."""
   106  
   107    def get(self):
   108      after_blob_ref = self.request.get('after')
   109      limit = max(1, min(1000, int(self.request.get('limit') or 1000)))
   110      query = Blob.all().order('__key__')
   111      if after_blob_ref:
   112        query.filter('__key__ >', db.Key.from_path(Blob.kind(), after_blob_ref))
   113      blob_ref_list = query.fetch(limit)
   114  
   115      self.response.headers['Content-Type'] = 'text/javascript'
   116      out = [
   117        '{\n'
   118        '    "blobs": ['
   119      ]
   120      if blob_ref_list:
   121        out.extend([
   122          '\n        ',
   123          ',\n        '.join(
   124            '{"blobRef": "%s", "size": %d}' %
   125            (b.key().name(), b.size) for b in blob_ref_list),
   126          '\n    ',
   127        ])
   128      if blob_ref_list and len(blob_ref_list) == limit:
   129        out.append(
   130          '],'
   131          '\n  "continueAfter": "%s"\n'
   132          '}' % blob_ref_list[-1].key().name())
   133      else:
   134        out.append(
   135          ']\n'
   136          '}'
   137        )
   138      self.response.out.write(''.join(out))
   139  
   140  
   141  class GetHandler(blobstore_handlers.BlobstoreDownloadHandler):
   142    """Gets a blob with the given ref."""
   143  
   144    def head(self, blob_ref):
   145      self.get(blob_ref)
   146  
   147    def get(self, blob_ref):
   148      blob = Blob.get_by_key_name(blob_ref)
   149      if not blob:
   150        self.error(404)
   151        return
   152      self.send_blob(blob.blob, 'application/octet-stream')
   153  
   154  
   155  class StatHandler(webapp.RequestHandler):
   156    """Handler to return a URL for a script to get an upload URL."""
   157  
   158    def stat_key(self):
   159      return "stat"
   160  
   161    def get(self):
   162      self.handle()
   163  
   164    def post(self):
   165      self.handle()
   166  
   167    def handle(self):
   168      if self.request.get('camliversion') != '1':
   169        self.response.headers['Content-Type'] = 'text/plain'
   170        self.response.out.write('Bad parameter: "camliversion"')
   171        self.response.set_status(400)
   172        return
   173  
   174      blob_ref_list = []
   175      for key, value in self.request.params.items():
   176        if not key.startswith('blob'):
   177          continue
   178        try:
   179          int(key[len('blob'):])
   180        except ValueError:
   181          logging.exception('Bad parameter: %s', key)
   182          self.response.headers['Content-Type'] = 'text/plain'
   183          self.response.out.write('Bad parameter: "%s"' % key)
   184          self.response.set_status(400)
   185          return
   186        else:
   187          blob_ref_list.append(value)
   188  
   189      key_name = self.stat_key()
   190  
   191      self.response.headers['Content-Type'] = 'text/javascript'
   192      out = [
   193        '{\n'
   194        '  "maxUploadSize": %d,\n'
   195        '  "uploadUrl": "%s",\n'
   196        '  "uploadUrlExpirationSeconds": 600,\n'
   197        '  "%s": [\n'
   198        % (config.MAX_UPLOAD_SIZE,
   199           blobstore.create_upload_url('/upload_complete'),
   200           key_name)
   201      ]
   202  
   203      already_have = db.get([
   204          db.Key.from_path(Blob.kind(), b) for b in blob_ref_list])
   205      if already_have:
   206        out.extend([
   207          '\n        ',
   208          ',\n        '.join(
   209            '{"blobRef": "%s", "size": %d}' %
   210            (b.key().name(), b.size) for b in already_have if b is not None),
   211          '\n    ',
   212        ])
   213      out.append(
   214        ']\n'
   215        '}'
   216      )
   217      self.response.out.write(''.join(out))
   218  
   219  
   220  class PostUploadHandler(StatHandler):
   221  
   222    def stat_key(self):
   223      return "received"
   224  
   225  
   226  class UploadHandler(blobstore_handlers.BlobstoreUploadHandler):
   227    """Handle blobstore post, as forwarded by notification agent."""
   228  
   229    def compute_blob_ref(self, hash_func, blob_key):
   230      """Computes the blob ref for a blob stored using the given hash function.
   231  
   232      Args:
   233        hash_func: The name of the hash function (sha1, md5)
   234        blob_key: The BlobKey of the App Engine blob containing the blob's data.
   235  
   236      Returns:
   237        A newly computed blob_ref for the data.
   238      """
   239      hasher = hashlib.new(hash_func)
   240      last_index = 0
   241      while True:
   242        data = blobstore.fetch_data(
   243            blob_key, last_index, last_index + blobstore.MAX_BLOB_FETCH_SIZE - 1)
   244        if not data:
   245          break
   246        hasher.update(data)
   247        last_index += len(data)
   248  
   249      return '%s-%s' % (hash_func, hasher.hexdigest())
   250  
   251    def store_blob(self, blob_ref, blob_info, error_messages):
   252      """Store blob information.
   253  
   254      Writes a Blob to the datastore for the uploaded file.
   255  
   256      Args:
   257        blob_ref: The file that was uploaded.
   258        upload_file: List of BlobInfo records representing the uploads.
   259        error_messages: Empty list for storing error messages to report to user.
   260      """
   261      if not blob_ref.startswith('sha1-'):
   262        error_messages.append('Only sha1 supported for now.')
   263        return
   264  
   265      if len(blob_ref) != (len('sha1-') + 40):
   266        error_messages.append('Bogus blobRef.')
   267        return
   268  
   269      found_blob_ref = self.compute_blob_ref('sha1', blob_info.key())
   270      if blob_ref != found_blob_ref:
   271        error_messages.append('Found blob ref %s, expected %s' %
   272                              (found_blob_ref, blob_ref))
   273        return
   274  
   275      def txn():
   276        logging.info('Saving blob "%s" with size %d', blob_ref, blob_info.size)
   277        blob = Blob(key_name=blob_ref, blob=blob_info.key(), size=blob_info.size)
   278        blob.put()
   279      db.run_in_transaction(txn)
   280  
   281    def post(self):
   282      """Do upload post."""
   283      error_messages = []
   284      blob_info_dict = {}
   285  
   286      for key, value in self.request.params.items():
   287        if isinstance(value, cgi.FieldStorage):
   288          if 'blob-key' in value.type_options:
   289            blob_info = blobstore.parse_blob_info(value)
   290            blob_info_dict[value.name] = blob_info
   291            logging.info("got blob: %s" % value.name)
   292            self.store_blob(value.name, blob_info, error_messages)
   293  
   294      if error_messages:
   295        logging.error('Upload errors: %r', error_messages)
   296        blobstore.delete(blob_info_dict.values())
   297        self.response.set_status(303)
   298        # TODO: fix up this format
   299        self.response.headers.add_header("Location", '/error?%s' % '&'.join(
   300            'error_message=%s' % urllib.quote(m) for m in error_messages))
   301      else:
   302        query = ['/nonstandard/upload_complete?camliversion=1']
   303        query.extend('blob%d=%s' % (i + 1, k)
   304                     for i, k in enumerate(blob_info_dict.iterkeys()))
   305        self.response.set_status(303)
   306        self.response.headers.add_header("Location", str('&'.join(query)))
   307  
   308  
   309  class ErrorHandler(webapp.RequestHandler):
   310    """The blob put failed."""
   311  
   312    def get(self):
   313      self.response.headers['Content-Type'] = 'text/plain'
   314      self.response.out.write('\n'.join(self.request.get_all('error_message')))
   315      self.response.set_status(400)
   316  
   317  
   318  class DebugUploadForm(webapp.RequestHandler):
   319    def get(self):
   320          self.response.headers['Content-Type'] = 'text/html'
   321          uploadurl = blobstore.create_upload_url('/upload_complete')
   322          self.response.out.write('<body><form method="post" enctype="multipart/form-data" action="%s">' % uploadurl)
   323          self.response.out.write('<input type="file" name="sha1-f628050e63819347a095645ad9ae697415664f0">')
   324          self.response.out.write('<input type="submit"></form></body>')
   325  
   326  
   327  APP = webapp.WSGIApplication(
   328    [
   329      ('/', HelloHandler),
   330      ('/debug/upform', DebugUploadForm),
   331      ('/camli/enumerate-blobs', ListHandler),
   332      ('/camli/stat', StatHandler),
   333      ('/camli/([^/]+)', GetHandler),
   334      ('/nonstandard/upload_complete', PostUploadHandler),
   335      ('/upload_complete', UploadHandler),  # Admin only.
   336      ('/error', ErrorHandler),
   337    ],
   338    debug=True)
   339  
   340  
   341  def main():
   342    wsgiref.handlers.CGIHandler().run(APP)
   343  
   344  
   345  if __name__ == '__main__':
   346    main()