couchdbkit.designer.fs

1 # -*- coding: utf-8 - 2 # 3 # This file is part of couchdbkit released under the MIT license. 4 # See the NOTICE for more information. 5 6 from __future__ import with_statement 7 import base64 8 import copy 9 from hashlib import md5 10 import logging 11 import mimetypes 12 import os 13 import os.path 14 import re 15 16 from .. import client 17 from ..exceptions import ResourceNotFound, DesignerError, \ 18 BulkSaveError 19 from .macros import package_shows, package_views 20 from .. import utils 21 22 if os.name == 'nt':

23 - def _replace_backslash(name):

24 return name.replace("\\", "/")

25

26 - def _replace_slash(name):

27 return name.replace("/", "\\")

28 else:

29 - def _replace_backslash(name):

30 return name

31

32 - def _replace_slash(name):

33 return name

34 35 logger = logging.getLogger(__name__) 36

37 -class FSDoc(object):

38

39 - def __init__(self, path, create=False, docid=None, is_ddoc=True):

40 self.docdir = path 41 self.ignores = [] 42 self.is_ddoc = is_ddoc 43 44 ignorefile = os.path.join(path, '.couchappignore') 45 if os.path.exists(ignorefile): 46 # A .couchappignore file is a json file containing a 47 # list of regexps for things to skip 48 self.ignores = utils.json.load(open(ignorefile, 'r')) 49 if not docid: 50 docid = self.get_id() 51 self.docid = docid 52 self._doc = {'_id': self.docid} 53 if create: 54 self.create()

55

56 - def get_id(self):

57 """ 58 if there is an _id file, docid is extracted from it, 59 else we take the current folder name. 60 """ 61 idfile = os.path.join(self.docdir, '_id') 62 if os.path.exists(idfile): 63 docid = utils.read_file(idfile).split("\n")[0].strip() 64 if docid: return docid 65 if self.is_ddoc: 66 return "_design/%s" % os.path.split(self.docdir)[1] 67 else: 68 return os.path.split(self.docdir)[1]

69

70 - def __repr__(self):

71 return "<%s (%s/%s)>" % (self.__class__.__name__, self.docdir, self.docid)

72

73 - def __str__(self):

74 return utils.json.dumps(self.doc())

75

76 - def create(self):

77 if not os.path.isdir(self.docdir): 78 logger.error("%s directory doesn't exist." % self.docdir) 79 80 rcfile = os.path.join(self.docdir, '.couchapprc') 81 if not os.path.isfile(rcfile): 82 utils.write_json(rcfile, {}) 83 else: 84 logger.warning("CouchApp already initialized in %s." % self.docdir)

85

86 - def push(self, dbs, atomic=True, force=False):

87 """Push a doc to a list of database `dburls`. If noatomic is true 88 each attachments will be sent one by one.""" 89 for db in dbs: 90 if atomic: 91 doc = self.doc(db, force=force) 92 db.save_doc(doc, force_update=True) 93 else: 94 doc = self.doc(db, with_attachments=False, force=force) 95 db.save_doc(doc, force_update=True) 96 97 attachments = doc.get('_attachments') or {} 98 99 for name, filepath in self.attachments(): 100 if name not in attachments: 101 logger.debug("attach %s " % name) 102 db.put_attachment(doc, open(filepath, "r"), 103 name=name) 104 105 logger.debug("%s/%s had been pushed from %s" % (db.uri, 106 self.docid, self.docdir))

107 108

109 - def attachment_stub(self, name, filepath):

110 att = {} 111 with open(filepath, "rb") as f: 112 re_sp = re.compile('\s') 113 att = { 114 "data": re_sp.sub('',base64.b64encode(f.read())), 115 "content_type": ';'.join(filter(None, 116 mimetypes.guess_type(name))) 117 } 118 119 return att

120

121 - def doc(self, db=None, with_attachments=True, force=False):

122 """ Function to reetrieve document object from 123 document directory. If `with_attachments` is True 124 attachments will be included and encoded""" 125 126 manifest = [] 127 objects = {} 128 signatures = {} 129 attachments = {} 130 131 self._doc = {'_id': self.docid} 132 133 # get designdoc 134 self._doc.update(self.dir_to_fields(self.docdir, manifest=manifest)) 135 136 if not 'couchapp' in self._doc: 137 self._doc['couchapp'] = {} 138 139 self.olddoc = {} 140 if db is not None: 141 try: 142 self.olddoc = db.open_doc(self._doc['_id']) 143 attachments = self.olddoc.get('_attachments') or {} 144 self._doc.update({'_rev': self.olddoc['_rev']}) 145 except ResourceNotFound: 146 self.olddoc = {} 147 148 if 'couchapp' in self.olddoc: 149 old_signatures = self.olddoc['couchapp'].get('signatures', 150 {}) 151 else: 152 old_signatures = {} 153 154 for name, filepath in self.attachments(): 155 signatures[name] = utils.sign_file(filepath) 156 if with_attachments and not old_signatures: 157 logger.debug("attach %s " % name) 158 attachments[name] = self.attachment_stub(name, filepath) 159 160 if old_signatures: 161 for name, signature in old_signatures.items(): 162 cursign = signatures.get(name) 163 if not cursign: 164 logger.debug("detach %s " % name) 165 del attachments[name] 166 elif cursign != signature: 167 logger.debug("detach %s " % name) 168 del attachments[name] 169 else: 170 continue 171 172 if with_attachments: 173 for name, filepath in self.attachments(): 174 if old_signatures.get(name) != signatures.get(name) or force: 175 logger.debug("attach %s " % name) 176 attachments[name] = self.attachment_stub(name, filepath) 177 178 self._doc['_attachments'] = attachments 179 180 self._doc['couchapp'].update({ 181 'manifest': manifest, 182 'objects': objects, 183 'signatures': signatures 184 }) 185 186 187 if self.docid.startswith('_design/'): # process macros 188 for funs in ['shows', 'lists', 'updates', 'filters', 189 'spatial']: 190 if funs in self._doc: 191 package_shows(self._doc, self._doc[funs], self.docdir, 192 objects) 193 194 if 'validate_doc_update' in self._doc: 195 tmp_dict = dict(validate_doc_update=self._doc[ 196 "validate_doc_update"]) 197 package_shows( self._doc, tmp_dict, self.docdir, 198 objects) 199 self._doc.update(tmp_dict) 200 201 if 'views' in self._doc: 202 # clean views 203 # we remove empty views and malformed from the list 204 # of pushed views. We also clean manifest 205 views = {} 206 dmanifest = {} 207 for i, fname in enumerate(manifest): 208 if fname.startswith("views/") and fname != "views/": 209 name, ext = os.path.splitext(fname) 210 if name.endswith('/'): 211 name = name[:-1] 212 dmanifest[name] = i 213 214 for vname, value in self._doc['views'].iteritems(): 215 if value and isinstance(value, dict): 216 views[vname] = value 217 else: 218 del manifest[dmanifest["views/%s" % vname]] 219 self._doc['views'] = views 220 package_views(self._doc,self._doc["views"], self.docdir, 221 objects) 222 223 if "fulltext" in self._doc: 224 package_views(self._doc,self._doc["fulltext"], self.docdir, 225 objects) 226 227 228 return self._doc

229

230 - def check_ignore(self, item):

231 for i in self.ignores: 232 match = re.match(i, item) 233 if match: 234 logger.debug("ignoring %s" % item) 235 return True 236 return False

237

238 - def dir_to_fields(self, current_dir='', depth=0, 239 manifest=[]):

240 """ process a directory and get all members """ 241 242 fields={} 243 if not current_dir: 244 current_dir = self.docdir 245 for name in os.listdir(current_dir): 246 current_path = os.path.join(current_dir, name) 247 rel_path = _replace_backslash(utils.relpath(current_path, self.docdir)) 248 if name.startswith("."): 249 continue 250 elif self.check_ignore(name): 251 continue 252 elif depth == 0 and name.startswith('_'): 253 # files starting with "_" are always "special" 254 continue 255 elif name == '_attachments': 256 continue 257 elif depth == 0 and (name == 'couchapp' or name == 'couchapp.json'): 258 # we are in app_meta 259 if name == "couchapp": 260 manifest.append('%s/' % rel_path) 261 content = self.dir_to_fields(current_path, 262 depth=depth+1, manifest=manifest) 263 else: 264 manifest.append(rel_path) 265 content = utils.read_json(current_path) 266 if not isinstance(content, dict): 267 content = { "meta": content } 268 if 'signatures' in content: 269 del content['signatures'] 270 271 if 'manifest' in content: 272 del content['manifest'] 273 274 if 'objects' in content: 275 del content['objects'] 276 277 if 'length' in content: 278 del content['length'] 279 280 if 'couchapp' in fields: 281 fields['couchapp'].update(content) 282 else: 283 fields['couchapp'] = content 284 elif os.path.isdir(current_path): 285 manifest.append('%s/' % rel_path) 286 fields[name] = self.dir_to_fields(current_path, 287 depth=depth+1, manifest=manifest) 288 else: 289 logger.debug("push %s" % rel_path) 290 291 content = '' 292 if name.endswith('.json'): 293 try: 294 content = utils.read_json(current_path) 295 except ValueError: 296 logger.error("Json invalid in %s" % current_path) 297 else: 298 try: 299 content = utils.read_file(current_path).strip() 300 except UnicodeDecodeError: 301 logger.warning("%s isn't encoded in utf8" % current_path) 302 content = utils.read_file(current_path, utf8=False) 303 try: 304 content.encode('utf-8') 305 except UnicodeError: 306 logger.warning( 307 "plan B didn't work, %s is a binary" % current_path) 308 logger.warning("use plan C: encode to base64") 309 content = "base64-encoded;%s" % base64.b64encode( 310 content) 311 312 313 # remove extension 314 name, ext = os.path.splitext(name) 315 if name in fields: 316 logger.warning( 317 "%(name)s is already in properties. Can't add (%(fqn)s)" % { 318 "name": name, "fqn": rel_path }) 319 else: 320 manifest.append(rel_path) 321 fields[name] = content 322 return fields

323

324 - def _process_attachments(self, path, vendor=None):

325 """ the function processing directory to yeld 326 attachments. """ 327 if os.path.isdir(path): 328 for root, dirs, files in os.walk(path): 329 for dirname in dirs: 330 if dirname.startswith('.'): 331 dirs.remove(dirname) 332 elif self.check_ignore(dirname): 333 dirs.remove(dirname) 334 if files: 335 for filename in files: 336 if filename.startswith('.'): 337 continue 338 elif self.check_ignore(filename): 339 continue 340 else: 341 filepath = os.path.join(root, filename) 342 name = utils.relpath(filepath, path) 343 if vendor is not None: 344 name = os.path.join('vendor', vendor, name) 345 name = _replace_backslash(name) 346 yield (name, filepath)

347

348 - def attachments(self):

349 """ This function yield a tuple (name, filepath) corresponding 350 to each attachment (vendor included) in the couchapp. `name` 351 is the name of attachment in `_attachments` member and `filepath` 352 the path to the attachment on the disk. 353 354 attachments are processed later to allow us to send attachments inline 355 or one by one. 356 """ 357 # process main attachments 358 attachdir = os.path.join(self.docdir, "_attachments") 359 for attachment in self._process_attachments(attachdir): 360 yield attachment 361 vendordir = os.path.join(self.docdir, 'vendor') 362 if not os.path.isdir(vendordir): 363 logger.debug("%s don't exist" % vendordir) 364 return 365 366 for name in os.listdir(vendordir): 367 current_path = os.path.join(vendordir, name) 368 if os.path.isdir(current_path): 369 attachdir = os.path.join(current_path, '_attachments') 370 if os.path.isdir(attachdir): 371 for attachment in self._process_attachments(attachdir, 372 vendor=name): 373 yield attachment

374

375 - def index(self, dburl, index):

376 if index is not None: 377 return "%s/%s/%s" % (dburl, self.docid, index) 378 elif os.path.isfile(os.path.join(self.docdir, "_attachments", 379 'index.html')): 380 return "%s/%s/index.html" % (dburl, self.docid) 381 return False

382

383 -def document(path, create=False, docid=None, is_ddoc=True):

384 """ simple function to retrive a doc object from filesystem """ 385 return FSDoc(path, create=create, docid=docid, is_ddoc=is_ddoc)

386

387 -def push(path, dbs, atomic=True, force=False, docid=None):

388 """ push a document from the fs to one or more dbs. Identic to 389 couchapp push command """ 390 if not isinstance(dbs, (list, tuple)): 391 dbs = [dbs] 392 393 doc = document(path, create=False, docid=docid) 394 doc.push(dbs, atomic=atomic, force=force) 395 docspath = os.path.join(path, '_docs') 396 if os.path.exists(docspath): 397 pushdocs(docspath, dbs, atomic=atomic)

398

399 -def pushapps(path, dbs, atomic=True, export=False, couchapprc=False):

400 """ push all couchapps in one folder like couchapp pushapps command 401 line """ 402 if not isinstance(dbs, (list, tuple)): 403 dbs = [dbs] 404 405 apps = [] 406 for d in os.listdir(path): 407 appdir = os.path.join(path, d) 408 if os.path.isdir(appdir): 409 if couchapprc and not os.path.isfile(os.path.join(appdir, 410 '.couchapprc')): 411 continue 412 doc = document(appdir) 413 if not atomic: 414 doc.push(dbs, atomic=False) 415 else: 416 apps.append(doc) 417 if apps: 418 if export: 419 docs= [doc.doc() for doc in apps] 420 jsonobj = {'docs': docs} 421 return jsonobj 422 else: 423 for db in dbs: 424 docs = [] 425 docs = [doc.doc(db) for doc in apps] 426 try: 427 db.save_docs(docs) 428 except BulkSaveError, e: 429 docs1 = [] 430 for doc in e.errors: 431 try: 432 doc['_rev'] = db.last_rev(doc['_id']) 433 docs1.append(doc) 434 except ResourceNotFound: 435 pass 436 if docs1: 437 db.save_docs(docs1)

438 439

440 -def pushdocs(path, dbs, atomic=True, export=False):

441 """ push multiple docs in a path """ 442 if not isinstance(dbs, (list, tuple)): 443 dbs = [dbs] 444 445 docs = [] 446 for d in os.listdir(path): 447 docdir = os.path.join(path, d) 448 if docdir.startswith('.'): 449 continue 450 elif os.path.isfile(docdir): 451 if d.endswith(".json"): 452 doc = utils.read_json(docdir) 453 docid, ext = os.path.splitext(d) 454 doc.setdefault('_id', docid) 455 doc.setdefault('couchapp', {}) 456 if not atomic: 457 for db in dbs: 458 db.save_doc(doc, force_update=True) 459 else: 460 docs.append(doc) 461 else: 462 doc = document(docdir, is_ddoc=False) 463 if not atomic: 464 doc.push(dbs, atomic=False) 465 else: 466 docs.append(doc) 467 if docs: 468 if export: 469 docs1 = [] 470 for doc in docs: 471 if hasattr(doc, 'doc'): 472 docs1.append(doc.doc()) 473 else: 474 docs1.append(doc) 475 jsonobj = {'docs': docs1} 476 return jsonobj 477 else: 478 for db in dbs: 479 docs1 = [] 480 for doc in docs: 481 if hasattr(doc, 'doc'): 482 docs1.append(doc.doc(db)) 483 else: 484 newdoc = doc.copy() 485 try: 486 rev = db.last_rev(doc['_id']) 487 newdoc.update({'_rev': rev}) 488 except ResourceNotFound: 489 pass 490 docs1.append(newdoc) 491 try: 492 db.save_docs(docs1) 493 except BulkSaveError, e: 494 # resolve conflicts 495 docs1 = [] 496 for doc in e.errors: 497 try: 498 doc['_rev'] = db.last_rev(doc['_id']) 499 docs1.append(doc) 500 except ResourceNotFound: 501 pass 502 if docs1: 503 db.save_docs(docs1)

504

505 -def clone(db, docid, dest=None, rev=None):

506 """ 507 Clone a CouchDB document to the fs. 508 509 """ 510 if not dest: 511 dest = docid 512 513 path = os.path.normpath(os.path.join(os.getcwd(), dest)) 514 if not os.path.exists(path): 515 os.makedirs(path) 516 517 if not rev: 518 doc = db.open_doc(docid) 519 else: 520 doc = db.open_doc(docid, rev=rev) 521 docid = doc['_id'] 522 523 524 metadata = doc.get('couchapp', {}) 525 526 # get manifest 527 manifest = metadata.get('manifest', {}) 528 529 # get signatures 530 signatures = metadata.get('signatures', {}) 531 532 # get objects refs 533 objects = metadata.get('objects', {}) 534 535 # create files from manifest 536 if manifest: 537 for filename in manifest: 538 logger.debug("clone property: %s" % filename) 539 filepath = os.path.join(path, filename) 540 if filename.endswith('/'): 541 if not os.path.isdir(filepath): 542 os.makedirs(filepath) 543 elif filename == "couchapp.json": 544 continue 545 else: 546 parts = utils.split_path(filename) 547 fname = parts.pop() 548 v = doc 549 while 1: 550 try: 551 for key in parts: 552 v = v[key] 553 except KeyError: 554 break 555 # remove extension 556 last_key, ext = os.path.splitext(fname) 557 558 # make sure key exist 559 try: 560 content = v[last_key] 561 except KeyError: 562 break 563 564 565 if isinstance(content, basestring): 566 _ref = md5(utils.to_bytestring(content)).hexdigest() 567 if objects and _ref in objects: 568 content = objects[_ref] 569 570 if content.startswith('base64-encoded;'): 571 content = base64.b64decode(content[15:]) 572 573 if fname.endswith('.json'): 574 content = utils.json.dumps(content).encode('utf-8') 575 576 del v[last_key] 577 578 # make sure file dir have been created 579 filedir = os.path.dirname(filepath) 580 if not os.path.isdir(filedir): 581 os.makedirs(filedir) 582 583 utils.write_content(filepath, content) 584 585 # remove the key from design doc 586 temp = doc 587 for key2 in parts: 588 if key2 == key: 589 if not temp[key2]: 590 del temp[key2] 591 break 592 temp = temp[key2] 593 594 595 # second pass for missing key or in case 596 # manifest isn't in app 597 for key in doc.iterkeys(): 598 if key.startswith('_'): 599 continue 600 elif key in ('couchapp'): 601 app_meta = copy.deepcopy(doc['couchapp']) 602 if 'signatures' in app_meta: 603 del app_meta['signatures'] 604 if 'manifest' in app_meta: 605 del app_meta['manifest'] 606 if 'objects' in app_meta: 607 del app_meta['objects'] 608 if 'length' in app_meta: 609 del app_meta['length'] 610 if app_meta: 611 couchapp_file = os.path.join(path, 'couchapp.json') 612 utils.write_json(couchapp_file, app_meta) 613 elif key in ('views'): 614 vs_dir = os.path.join(path, key) 615 if not os.path.isdir(vs_dir): 616 os.makedirs(vs_dir) 617 for vsname, vs_item in doc[key].iteritems(): 618 vs_item_dir = os.path.join(vs_dir, vsname) 619 if not os.path.isdir(vs_item_dir): 620 os.makedirs(vs_item_dir) 621 for func_name, func in vs_item.iteritems(): 622 filename = os.path.join(vs_item_dir, '%s.js' % 623 func_name) 624 utils.write_content(filename, func) 625 logger.warning("clone view not in manifest: %s" % filename) 626 elif key in ('shows', 'lists', 'filter', 'update'): 627 showpath = os.path.join(path, key) 628 if not os.path.isdir(showpath): 629 os.makedirs(showpath) 630 for func_name, func in doc[key].iteritems(): 631 filename = os.path.join(showpath, '%s.js' % 632 func_name) 633 utils.write_content(filename, func) 634 logger.warning( 635 "clone show or list not in manifest: %s" % filename) 636 else: 637 filedir = os.path.join(path, key) 638 if os.path.exists(filedir): 639 continue 640 else: 641 logger.warning("clone property not in manifest: %s" % key) 642 if isinstance(doc[key], (list, tuple,)): 643 utils.write_json(filedir + ".json", doc[key]) 644 elif isinstance(doc[key], dict): 645 if not os.path.isdir(filedir): 646 os.makedirs(filedir) 647 for field, value in doc[key].iteritems(): 648 fieldpath = os.path.join(filedir, field) 649 if isinstance(value, basestring): 650 if value.startswith('base64-encoded;'): 651 value = base64.b64decode(content[15:]) 652 utils.write_content(fieldpath, value) 653 else: 654 utils.write_json(fieldpath + '.json', value) 655 else: 656 value = doc[key] 657 if not isinstance(value, basestring): 658 value = str(value) 659 utils.write_content(filedir, value) 660 661 # save id 662 idfile = os.path.join(path, '_id') 663 utils.write_content(idfile, doc['_id']) 664 665 utils.write_json(os.path.join(path, '.couchapprc'), {}) 666 667 if '_attachments' in doc: # process attachments 668 attachdir = os.path.join(path, '_attachments') 669 if not os.path.isdir(attachdir): 670 os.makedirs(attachdir) 671 672 for filename in doc['_attachments'].iterkeys(): 673 if filename.startswith('vendor'): 674 attach_parts = utils.split_path(filename) 675 vendor_attachdir = os.path.join(path, attach_parts.pop(0), 676 attach_parts.pop(0), '_attachments') 677 filepath = os.path.join(vendor_attachdir, *attach_parts) 678 else: 679 filepath = os.path.join(attachdir, filename) 680 filepath = _replace_slash(filepath) 681 currentdir = os.path.dirname(filepath) 682 if not os.path.isdir(currentdir): 683 os.makedirs(currentdir) 684 685 if signatures.get(filename) != utils.sign_file(filepath): 686 stream = db.fetch_attachment(docid, filename, stream=True) 687 with open(filepath, 'wb') as f: 688 for chunk in stream: 689 f.write(chunk) 690 logger.debug("clone attachment: %s" % filename) 691 692 logger.debug("%s/%s cloned in %s" % (db.uri, docid, dest))

693

694 -def clone_design_doc(source, dest, rev=None):

695 """ Clone a design document from it's url like couchapp does. 696 """ 697 try: 698 dburl, docid = source.split('_design/') 699 except ValueError: 700 raise DesignerError("%s isn't a valid source" % source) 701 702 db = client.Database(dburl[:-1], create=False) 703 clone(db, docid, dest, rev=rev)

704

Source Code for Module couchdbkit.designer.fs