github.com/olivere/camlistore@v0.0.0-20140121221811-1b7ac2da0199/server/gae-py-blobserver/main.py (about) 1 #!/usr/bin/env python 2 # 3 # Camlistore blob server for App Engine. 4 # 5 # Derived from Brad's Brackup-gae utility: 6 # http://github.com/bradfitz/brackup-gae-server 7 # 8 # Copyright 2010 Google Inc. 9 # 10 # Licensed under the Apache License, Version 2.0 (the "License"); 11 # you may not use this file except in compliance with the License. 12 # You may obtain a copy of the License at 13 # 14 # http://www.apache.org/licenses/LICENSE-2.0 15 # 16 # Unless required by applicable law or agreed to in writing, software 17 # distributed under the License is distributed on an "AS IS" BASIS, 18 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 # See the License for the specific language governing permissions and 20 # limitations under the License. 21 # 22 23 """Upload server for camlistore. 24 25 To test: 26 27 # Stat -- 200 response 28 curl -v \ 29 -d camliversion=1 \ 30 http://localhost:8080/camli/stat 31 32 # Upload -- 200 response 33 curl -v -L \ 34 -F sha1-126249fd8c18cbb5312a5705746a2af87fba9538=@./test_data.txt \ 35 <the url returned by stat> 36 37 # Put with bad blob_ref parameter -- 400 response 38 curl -v -L \ 39 -F sha1-22a7fdd575f4c3e7caa3a55cc83db8b8a6714f0f=@./test_data.txt \ 40 <the url returned by stat> 41 42 # Get present -- the blob 43 curl -v http://localhost:8080/camli/\ 44 sha1-126249fd8c18cbb5312a5705746a2af87fba9538 45 46 # Get missing -- 404 47 curl -v http://localhost:8080/camli/\ 48 sha1-22a7fdd575f4c3e7caa3a55cc83db8b8a6714f0f 49 50 # Check present -- 200 with only headers 51 curl -I http://localhost:8080/camli/\ 52 sha1-126249fd8c18cbb5312a5705746a2af87fba9538 53 54 # Check missing -- 404 with empty list response 55 curl -I http://localhost:8080/camli/\ 56 sha1-22a7fdd575f4c3e7caa3a55cc83db8b8a6714f0f 57 58 # List -- 200 with list of blobs (just one) 59 curl -v http://localhost:8080/camli/enumerate-blobs&limit=1 60 61 # List offset -- 200 with list of no blobs 62 curl -v http://localhost:8080/camli/enumerate-blobs?after=\ 63 sha1-126249fd8c18cbb5312a5705746a2af87fba9538 64 65 """ 66 67 import cgi 68 import hashlib 69 import logging 70 import urllib 71 import wsgiref.handlers 72 73 from google.appengine.ext import blobstore 74 from google.appengine.ext import db 75 from google.appengine.ext import webapp 76 from google.appengine.ext.webapp import blobstore_handlers 77 78 import config 79 80 81 class Blob(db.Model): 82 """Some content-addressable blob. 83 84 The key is the algorithm, dash, and the lowercase hex digest: 85 "sha1-f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" 86 """ 87 88 # The actual bytes. 89 blob = blobstore.BlobReferenceProperty(indexed=False) 90 91 # Size. (already in the blobinfo, but denormalized for speed) 92 size = db.IntegerProperty(indexed=False) 93 94 95 class HelloHandler(webapp.RequestHandler): 96 """Present ourselves to the world.""" 97 98 def get(self): 99 self.response.out.write('Hello! This is an AppEngine Camlistore ' 100 'blob server.<p>') 101 self.response.out.write('<a href=js/index.html>js frontend</a>') 102 103 104 class ListHandler(webapp.RequestHandler): 105 """Return chunks that the server has.""" 106 107 def get(self): 108 after_blob_ref = self.request.get('after') 109 limit = max(1, min(1000, int(self.request.get('limit') or 1000))) 110 query = Blob.all().order('__key__') 111 if after_blob_ref: 112 query.filter('__key__ >', db.Key.from_path(Blob.kind(), after_blob_ref)) 113 blob_ref_list = query.fetch(limit) 114 115 self.response.headers['Content-Type'] = 'text/javascript' 116 out = [ 117 '{\n' 118 ' "blobs": [' 119 ] 120 if blob_ref_list: 121 out.extend([ 122 '\n ', 123 ',\n '.join( 124 '{"blobRef": "%s", "size": %d}' % 125 (b.key().name(), b.size) for b in blob_ref_list), 126 '\n ', 127 ]) 128 if blob_ref_list and len(blob_ref_list) == limit: 129 out.append( 130 '],' 131 '\n "continueAfter": "%s"\n' 132 '}' % blob_ref_list[-1].key().name()) 133 else: 134 out.append( 135 ']\n' 136 '}' 137 ) 138 self.response.out.write(''.join(out)) 139 140 141 class GetHandler(blobstore_handlers.BlobstoreDownloadHandler): 142 """Gets a blob with the given ref.""" 143 144 def head(self, blob_ref): 145 self.get(blob_ref) 146 147 def get(self, blob_ref): 148 blob = Blob.get_by_key_name(blob_ref) 149 if not blob: 150 self.error(404) 151 return 152 self.send_blob(blob.blob, 'application/octet-stream') 153 154 155 class StatHandler(webapp.RequestHandler): 156 """Handler to return a URL for a script to get an upload URL.""" 157 158 def stat_key(self): 159 return "stat" 160 161 def get(self): 162 self.handle() 163 164 def post(self): 165 self.handle() 166 167 def handle(self): 168 if self.request.get('camliversion') != '1': 169 self.response.headers['Content-Type'] = 'text/plain' 170 self.response.out.write('Bad parameter: "camliversion"') 171 self.response.set_status(400) 172 return 173 174 blob_ref_list = [] 175 for key, value in self.request.params.items(): 176 if not key.startswith('blob'): 177 continue 178 try: 179 int(key[len('blob'):]) 180 except ValueError: 181 logging.exception('Bad parameter: %s', key) 182 self.response.headers['Content-Type'] = 'text/plain' 183 self.response.out.write('Bad parameter: "%s"' % key) 184 self.response.set_status(400) 185 return 186 else: 187 blob_ref_list.append(value) 188 189 key_name = self.stat_key() 190 191 self.response.headers['Content-Type'] = 'text/javascript' 192 out = [ 193 '{\n' 194 ' "maxUploadSize": %d,\n' 195 ' "uploadUrl": "%s",\n' 196 ' "uploadUrlExpirationSeconds": 600,\n' 197 ' "%s": [\n' 198 % (config.MAX_UPLOAD_SIZE, 199 blobstore.create_upload_url('/upload_complete'), 200 key_name) 201 ] 202 203 already_have = db.get([ 204 db.Key.from_path(Blob.kind(), b) for b in blob_ref_list]) 205 if already_have: 206 out.extend([ 207 '\n ', 208 ',\n '.join( 209 '{"blobRef": "%s", "size": %d}' % 210 (b.key().name(), b.size) for b in already_have if b is not None), 211 '\n ', 212 ]) 213 out.append( 214 ']\n' 215 '}' 216 ) 217 self.response.out.write(''.join(out)) 218 219 220 class PostUploadHandler(StatHandler): 221 222 def stat_key(self): 223 return "received" 224 225 226 class UploadHandler(blobstore_handlers.BlobstoreUploadHandler): 227 """Handle blobstore post, as forwarded by notification agent.""" 228 229 def compute_blob_ref(self, hash_func, blob_key): 230 """Computes the blob ref for a blob stored using the given hash function. 231 232 Args: 233 hash_func: The name of the hash function (sha1, md5) 234 blob_key: The BlobKey of the App Engine blob containing the blob's data. 235 236 Returns: 237 A newly computed blob_ref for the data. 238 """ 239 hasher = hashlib.new(hash_func) 240 last_index = 0 241 while True: 242 data = blobstore.fetch_data( 243 blob_key, last_index, last_index + blobstore.MAX_BLOB_FETCH_SIZE - 1) 244 if not data: 245 break 246 hasher.update(data) 247 last_index += len(data) 248 249 return '%s-%s' % (hash_func, hasher.hexdigest()) 250 251 def store_blob(self, blob_ref, blob_info, error_messages): 252 """Store blob information. 253 254 Writes a Blob to the datastore for the uploaded file. 255 256 Args: 257 blob_ref: The file that was uploaded. 258 upload_file: List of BlobInfo records representing the uploads. 259 error_messages: Empty list for storing error messages to report to user. 260 """ 261 if not blob_ref.startswith('sha1-'): 262 error_messages.append('Only sha1 supported for now.') 263 return 264 265 if len(blob_ref) != (len('sha1-') + 40): 266 error_messages.append('Bogus blobRef.') 267 return 268 269 found_blob_ref = self.compute_blob_ref('sha1', blob_info.key()) 270 if blob_ref != found_blob_ref: 271 error_messages.append('Found blob ref %s, expected %s' % 272 (found_blob_ref, blob_ref)) 273 return 274 275 def txn(): 276 logging.info('Saving blob "%s" with size %d', blob_ref, blob_info.size) 277 blob = Blob(key_name=blob_ref, blob=blob_info.key(), size=blob_info.size) 278 blob.put() 279 db.run_in_transaction(txn) 280 281 def post(self): 282 """Do upload post.""" 283 error_messages = [] 284 blob_info_dict = {} 285 286 for key, value in self.request.params.items(): 287 if isinstance(value, cgi.FieldStorage): 288 if 'blob-key' in value.type_options: 289 blob_info = blobstore.parse_blob_info(value) 290 blob_info_dict[value.name] = blob_info 291 logging.info("got blob: %s" % value.name) 292 self.store_blob(value.name, blob_info, error_messages) 293 294 if error_messages: 295 logging.error('Upload errors: %r', error_messages) 296 blobstore.delete(blob_info_dict.values()) 297 self.response.set_status(303) 298 # TODO: fix up this format 299 self.response.headers.add_header("Location", '/error?%s' % '&'.join( 300 'error_message=%s' % urllib.quote(m) for m in error_messages)) 301 else: 302 query = ['/nonstandard/upload_complete?camliversion=1'] 303 query.extend('blob%d=%s' % (i + 1, k) 304 for i, k in enumerate(blob_info_dict.iterkeys())) 305 self.response.set_status(303) 306 self.response.headers.add_header("Location", str('&'.join(query))) 307 308 309 class ErrorHandler(webapp.RequestHandler): 310 """The blob put failed.""" 311 312 def get(self): 313 self.response.headers['Content-Type'] = 'text/plain' 314 self.response.out.write('\n'.join(self.request.get_all('error_message'))) 315 self.response.set_status(400) 316 317 318 class DebugUploadForm(webapp.RequestHandler): 319 def get(self): 320 self.response.headers['Content-Type'] = 'text/html' 321 uploadurl = blobstore.create_upload_url('/upload_complete') 322 self.response.out.write('<body><form method="post" enctype="multipart/form-data" action="%s">' % uploadurl) 323 self.response.out.write('<input type="file" name="sha1-f628050e63819347a095645ad9ae697415664f0">') 324 self.response.out.write('<input type="submit"></form></body>') 325 326 327 APP = webapp.WSGIApplication( 328 [ 329 ('/', HelloHandler), 330 ('/debug/upform', DebugUploadForm), 331 ('/camli/enumerate-blobs', ListHandler), 332 ('/camli/stat', StatHandler), 333 ('/camli/([^/]+)', GetHandler), 334 ('/nonstandard/upload_complete', PostUploadHandler), 335 ('/upload_complete', UploadHandler), # Admin only. 336 ('/error', ErrorHandler), 337 ], 338 debug=True) 339 340 341 def main(): 342 wsgiref.handlers.CGIHandler().run(APP) 343 344 345 if __name__ == '__main__': 346 main()