Hatta Devel
view hatta.py @ 724:22b21a40da4e
make the html generated by the code highlighter validate -- use spans instead of divs
| author | Radomir Dopieralski <devel@sheep.art.pl> |
|---|---|
| date | Sun Jan 10 20:36:38 2010 +0100 (2010-01-10) |
| parents | 3b5177cf4780 |
| children | ecb0716ced56 |
line source
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
4 # @copyright: 2008-2009 Radomir Dopieralski <hatta@sheep.art.pl>
5 # @license: GNU GPL, see COPYING for details.
7 """
8 Hatta Wiki is a wiki engine designed to be used with Mercurial repositories.
9 It requires Mercurial and Werkzeug python modules.
11 Hatta's pages are just plain text files (and also images, binaries, etc.) in
12 some directory in your repository. For example, you can put it in your
13 project's "docs" directory to keep documentation. The files can be edited both
14 from the wiki or with a text editor -- in either case the changes committed to
15 the repository will appear in the recent changes and in page's history.
17 See hatta.py --help for usage.
18 """
20 import base64
21 import datetime
22 import difflib
23 import gettext
24 import itertools
25 import mimetypes
26 import os
27 import re
28 import sqlite3
29 import sys
30 import tempfile
31 import thread
32 import unicodedata
35 # Avoid WSGI errors, see http://mercurial.selenic.com/bts/issue1095
36 sys.stdout = sys.__stdout__
37 sys.stderr = sys.__stderr__
39 import werkzeug
40 import werkzeug.exceptions
41 import werkzeug.routing
43 try:
44 import Image
45 except ImportError:
46 Image = None
48 try:
49 import pygments
50 import pygments.util
51 import pygments.lexers
52 import pygments.formatters
53 import pygments.styles
54 except ImportError:
55 pygments = None
57 # Note: we have to set these before importing Mercurial
58 os.environ['HGENCODING'] = 'utf-8'
59 os.environ['HGMERGE'] = "internal:merge"
61 import mercurial.hg
62 import mercurial.ui
63 import mercurial.revlog
64 import mercurial.util
65 import mercurial.hgweb
67 __version__ = '1.3.3dev'
68 name = 'Hatta'
69 url = 'http://hatta-wiki.org/'
70 description = 'Wiki engine that lives in Mercurial repository.'
72 mimetypes.add_type('application/x-python', '.wsgi')
73 mimetypes.add_type('application/x-javascript', '.js')
74 mimetypes.add_type('text/x-rst', '.rst')
77 def external_link(addr):
78 """
79 Decide whether a link is absolute or internal.
81 >>> external_link('http://example.com')
82 True
83 >>> external_link('https://example.com')
84 True
85 >>> external_link('ftp://example.com')
86 True
87 >>> external_link('mailto:user@example.com')
88 True
89 >>> external_link('PageTitle')
90 False
91 >>> external_link(u'ąęśćUnicodePage')
92 False
94 """
96 return (addr.startswith('http://')
97 or addr.startswith('https://')
98 or addr.startswith('ftp://')
99 or addr.startswith('mailto:'))
102 class WikiConfig(object):
103 """
104 Responsible for reading and storing site configuration. Contains the
105 default settings.
107 >>> config = WikiConfig(port='2080')
108 >>> config.sanitize()
109 >>> config.get('port')
110 2080
111 """
113 default_filename = u'hatta.conf'
115 # Please see the bottom of the script for modifying these values.
117 def __init__(self, **kw):
118 self.config = dict(kw)
119 self.parse_environ()
121 def sanitize(self):
122 """
123 Convert options to their required types.
124 """
126 try:
127 self.config['port'] = int(self.get('port', 0))
128 except ValueError:
129 self.config['port'] = 8080
131 def parse_environ(self):
132 """Check the environment variables for options."""
134 prefix = 'HATTA_'
135 for key, value in os.environ.iteritems():
136 if key.startswith(prefix):
137 name = key[len(prefix):].lower()
138 self.config[name] = value
140 def parse_args(self):
141 """Check the commandline arguments for options."""
143 import optparse
145 self.options = []
146 parser = optparse.OptionParser()
148 def add(*args, **kw):
149 self.options.append(kw['dest'])
150 parser.add_option(*args, **kw)
152 add('-V', '--version', dest='show_version', default=False,
153 help='Display version and exit', action="store_true")
154 add('-d', '--pages-dir', dest='pages_path',
155 help='Store pages in DIR', metavar='DIR')
156 add('-t', '--cache-dir', dest='cache_path',
157 help='Store cache in DIR', metavar='DIR')
158 add('-i', '--interface', dest='interface',
159 help='Listen on interface INT', metavar='INT')
160 add('-p', '--port', dest='port', type='int',
161 help='Listen on port PORT', metavar='PORT')
162 add('-s', '--script-name', dest='script_name',
163 help='Override SCRIPT_NAME to NAME', metavar='NAME')
164 add('-n', '--site-name', dest='site_name',
165 help='Set the name of the site to NAME', metavar='NAME')
166 add('-m', '--front-page', dest='front_page',
167 help='Use PAGE as the front page', metavar='PAGE')
168 add('-e', '--encoding', dest='page_charset',
169 help='Use encoding ENC to read and write pages', metavar='ENC')
170 add('-c', '--config-file', dest='config_file',
171 help='Read configuration from FILE', metavar='FILE')
172 add('-l', '--language', dest='language',
173 help='Translate interface to LANG', metavar='LANG')
174 add('-r', '--read-only', dest='read_only',
175 help='Whether the wiki should be read-only', action="store_true")
176 add('-g', '--icon-page', dest='icon_page', metavar="PAGE",
177 help='Read icons graphics from PAGE.')
178 add('-w', '--hgweb', dest='hgweb',
179 help='Enable hgweb access to the repository', action="store_true")
180 add('-W', '--wiki-words', dest='wiki_words',
181 help='Enable WikiWord links', action="store_true")
182 add('-I', '--ignore-indent', dest='ignore_indent',
183 help='Treat indented lines as normal text', action="store_true")
184 add('-P', '--pygments-style', dest='pygments_style',
185 help='Use the STYLE pygments style for highlighting',
186 metavar='STYLE')
188 options, args = parser.parse_args()
189 for option, value in options.__dict__.iteritems():
190 if option in self.options:
191 if value is not None:
192 self.config[option] = value
194 def parse_files(self, files=None):
195 """Check the config files for options."""
197 import ConfigParser
199 if files is None:
200 files = [self.get('config_file', self.default_filename)]
201 parser = ConfigParser.SafeConfigParser()
202 parser.read(files)
203 for section in parser.sections():
204 for option, value in parser.items(section):
205 self.config[option] = value
207 def save_config(self, filename=None):
208 """Saves configuration to a given file."""
209 if filename is None:
210 filename = self.default_filename
212 import ConfigParser
213 parser = ConfigParser.RawConfigParser()
214 section = self.config['site_name']
215 parser.add_section(section)
216 for key, value in self.config.iteritems():
217 parser.set(section, str(key), str(value))
219 configfile = open(filename, 'wb')
220 try:
221 parser.write(configfile)
222 finally:
223 configfile.close()
225 def get(self, option, default_value=None):
226 """
227 Get the value of a config option or default if not set.
229 >>> config = WikiConfig(option=4)
230 >>> config.get("ziew", 3)
231 3
232 >>> config.get("ziew")
233 >>> config.get("ziew", "ziew")
234 'ziew'
235 >>> config.get("option")
236 4
237 """
239 return self.config.get(option, default_value)
241 def get_bool(self, option, default_value=False):
242 """
243 Like get, only convert the value to True or False.
244 """
246 value = self.get(option, default_value)
247 if value in (
248 1, True,
249 'True', 'true', 'TRUE',
250 '1',
251 'on', 'On', 'ON',
252 'yes', 'Yes', 'YES',
253 'enable', 'Enable', 'ENABLE',
254 'enabled', 'Enabled', 'ENABLED',
255 ):
256 return True
257 elif value in (
258 None, 0, False,
259 'False', 'false', 'FALSE',
260 '0',
261 'off', 'Off', 'OFF',
262 'no', 'No', 'NO',
263 'disable', 'Disable', 'DISABLE',
264 'disabled', 'Disabled', 'DISABLED',
265 ):
266 return False
267 else:
268 raise ValueError("expected boolean value")
270 def set(self, key, value):
271 self.config[key] = value
274 def locked_repo(func):
275 """A decorator for locking the repository when calling a method."""
277 def new_func(self, *args, **kwargs):
278 """Wrap the original function in locks."""
280 wlock = self.repo.wlock()
281 lock = self.repo.lock()
282 try:
283 func(self, *args, **kwargs)
284 finally:
285 lock.release()
286 wlock.release()
288 return new_func
290 class WikiStorage(object):
291 """
292 Provides means of storing wiki pages and keeping track of their
293 change history, using Mercurial repository as the storage method.
294 """
296 def __init__(self, path, charset=None):
297 """
298 Takes the path to the directory where the pages are to be kept.
299 If the directory doen't exist, it will be created. If it's inside
300 a Mercurial repository, that repository will be used, otherwise
301 a new repository will be created in it.
302 """
304 self.charset = charset or 'utf-8'
305 self.path = path
306 if not os.path.exists(self.path):
307 os.makedirs(self.path)
308 self.repo_path = self._find_repo_path(self.path)
309 try:
310 self.ui = mercurial.ui.ui(report_untrusted=False,
311 interactive=False, quiet=True)
312 except TypeError:
313 # Mercurial 1.3 changed the way we setup the ui object.
314 self.ui = mercurial.ui.ui()
315 self.ui.quiet = True
316 self.ui._report_untrusted = False
317 self.ui.setconfig('ui', 'interactive', False)
318 if self.repo_path is None:
319 self.repo_path = self.path
320 create = True
321 else:
322 create = False
323 self.repo_prefix = self.path[len(self.repo_path):].strip('/')
324 self._repos = {}
325 # Create the repository if needed.
326 mercurial.hg.repository(self.ui, self.repo_path, create=create)
328 def reopen(self):
329 """Close and reopen the repo, to make sure we are up to date."""
331 #self.repo = mercurial.hg.repository(self.ui, self.repo_path)
332 self._repos = {}
334 @property
335 def repo(self):
336 """Keep one open repository per thread."""
338 thread_id = thread.get_ident()
339 try:
340 return self._repos[thread_id]
341 except KeyError:
342 repo = mercurial.hg.repository(self.ui, self.repo_path)
343 self._repos[thread_id] = repo
344 return repo
346 def _find_repo_path(self, path):
347 """Go up the directory tree looking for a repository."""
349 while not os.path.isdir(os.path.join(path, ".hg")):
350 old_path, path = path, os.path.dirname(path)
351 if path == old_path:
352 return None
353 return path
355 def _file_path(self, title):
356 title = unicode(title).strip()
357 return os.path.join(self.path,
358 werkzeug.url_quote(title, safe=''))
360 def _title_to_file(self, title):
361 title = unicode(title).strip()
362 return os.path.join(self.repo_prefix,
363 werkzeug.url_quote(title, safe=''))
365 def _file_to_title(self, filename):
366 assert filename.startswith(self.repo_prefix)
367 name = filename[len(self.repo_prefix):].strip('/')
368 return werkzeug.url_unquote(name)
370 def __contains__(self, title):
371 if title:
372 file_path = self._file_path(title)
373 return os.path.isfile(file_path) and not os.path.islink(file_path)
375 def __iter__(self):
376 return self.all_pages()
378 def merge_changes(self, changectx, repo_file, text, user, parent):
379 """Commits and merges conflicting changes in the repository."""
381 tip_node = changectx.node()
382 filectx = changectx[repo_file].filectx(parent)
383 parent_node = filectx.changectx().node()
385 self.repo.dirstate.setparents(parent_node)
386 node = self._commit([repo_file], text, user)
388 partial = lambda filename: repo_file == filename
389 try:
390 unresolved = mercurial.merge.update(self.repo, tip_node,
391 True, True, partial)
392 msg = _(u'merge of edit conflict')
393 except mercurial.util.Abort:
394 unresolved = 1, 1, 1, 1
395 msg = _(u'failed merge of edit conflict')
396 self.repo.dirstate.setparents(tip_node, node)
397 # Mercurial 1.1 and later need updating the merge state
398 try:
399 mercurial.merge.mergestate(self.repo).mark(repo_file, "r")
400 except (AttributeError, KeyError):
401 pass
402 return msg
404 @locked_repo
405 def save_file(self, title, file_name, author=u'', comment=u'', parent=None):
406 """Save an existing file as specified page."""
408 user = author.encode('utf-8') or _(u'anon').encode('utf-8')
409 text = comment.encode('utf-8') or _(u'comment').encode('utf-8')
410 repo_file = self._title_to_file(title)
411 file_path = self._file_path(title)
412 if os.path.islink(file_path):
413 raise werkzeug.exceptions.Forbidden(u"Can't edit symbolic links")
414 mercurial.util.rename(file_name, file_path)
415 changectx = self._changectx()
416 try:
417 filectx_tip = changectx[repo_file]
418 current_page_rev = filectx_tip.filerev()
419 except mercurial.revlog.LookupError:
420 self.repo.add([repo_file])
421 current_page_rev = -1
422 if parent is not None and current_page_rev != parent:
423 msg = self.merge_changes(changectx, repo_file, text, user, parent)
424 user = '<wiki>'
425 text = msg.encode('utf-8')
426 self._commit([repo_file], text, user)
429 def _commit(self, files, text, user):
430 try:
431 return self.repo.commit(files=files, text=text, user=user,
432 force=True, empty_ok=True)
433 except TypeError:
434 # Mercurial 1.3 doesn't accept empty_ok or files parameter
435 match = mercurial.match.exact(self.repo_path, '', list(files))
436 return self.repo.commit(match=match, text=text, user=user,
437 force=True)
440 def save_data(self, title, data, author=u'', comment=u'', parent=None):
441 """Save data as specified page."""
443 try:
444 temp_path = tempfile.mkdtemp(dir=self.path)
445 file_path = os.path.join(temp_path, 'saved')
446 f = open(file_path, "wb")
447 f.write(data)
448 f.close()
449 self.save_file(title, file_path, author, comment, parent)
450 finally:
451 try:
452 os.unlink(file_path)
453 except OSError:
454 pass
455 try:
456 os.rmdir(temp_path)
457 except OSError:
458 pass
460 def save_text(self, title, text, author=u'', comment=u'', parent=None):
461 """Save text as specified page, encoded to charset."""
463 data = text.encode(self.charset)
464 self.save_data(title, data, author, comment, parent)
466 def page_text(self, title):
467 """Read unicode text of a page."""
469 data = self.open_page(title).read()
470 text = unicode(data, self.charset, 'replace')
471 return text
473 def page_lines(self, page):
474 for data in page.xreadlines():
475 yield unicode(data, self.charset, 'replace')
477 @locked_repo
478 def delete_page(self, title, author=u'', comment=u''):
479 user = author.encode('utf-8') or 'anon'
480 text = comment.encode('utf-8') or 'deleted'
481 repo_file = self._title_to_file(title)
482 file_path = self._file_path(title)
483 if os.path.islink(file_path):
484 raise werkzeug.exceptions.Forbidden(u"Can't edit symbolic links")
485 try:
486 os.unlink(file_path)
487 except OSError:
488 pass
489 self.repo.remove([repo_file])
490 self._commit([repo_file], text, user)
492 def open_page(self, title):
493 """Open the page and return a file-like object with its contents."""
495 file_path = self._file_path(title)
496 if os.path.islink(file_path):
497 raise werkzeug.exceptions.Forbidden(u"Can't read symbolic links")
498 try:
499 return open(file_path, "rb")
500 except IOError:
501 raise werkzeug.exceptions.NotFound()
503 def page_file_meta(self, title):
504 """Get page's inode number, size and last modification time."""
506 try:
507 (st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size,
508 st_atime, st_mtime, st_ctime) = os.stat(self._file_path(title))
509 except OSError:
510 return 0, 0, 0
511 return st_ino, st_size, st_mtime
513 def page_meta(self, title):
514 """Get page's revision, date, last editor and his edit comment."""
516 filectx_tip = self._find_filectx(title)
517 if filectx_tip is None:
518 raise werkzeug.exceptions.NotFound()
519 #return -1, None, u'', u''
520 rev = filectx_tip.filerev()
521 filectx = filectx_tip.filectx(rev)
522 date = datetime.datetime.fromtimestamp(filectx.date()[0])
523 author = unicode(filectx.user(), "utf-8",
524 'replace').split('<')[0].strip()
525 comment = unicode(filectx.description(), "utf-8", 'replace')
526 return rev, date, author, comment
528 def repo_revision(self):
529 """Give the latest revision of the repository."""
531 return self._changectx().rev()
533 def page_mime(self, title):
534 """
535 Guess page's mime type ased on corresponding file name.
536 Default ot text/x-wiki for files without an extension.
538 >>> page_mime('something.txt')
539 'text/plain'
540 >>> page_mime('SomePage')
541 'text/x-wiki'
542 >>> page_mime(u'ąęśUnicodePage')
543 'text/x-wiki'
544 >>> page_mime('image.png')
545 'image/png'
546 >>> page_mime('style.css')
547 'text/css'
548 >>> page_mime('archive.tar.gz')
549 'archive/gzip'
550 """
552 addr = self._file_path(title)
553 mime, encoding = mimetypes.guess_type(addr, strict=False)
554 if encoding:
555 mime = 'archive/%s' % encoding
556 if mime is None:
557 mime = 'text/x-wiki'
558 return mime
560 def _changectx(self):
561 """Get the changectx of the tip."""
563 try:
564 # This is for Mercurial 1.0
565 return self.repo.changectx()
566 except TypeError:
567 # Mercurial 1.3 (and possibly earlier) needs an argument
568 return self.repo.changectx('tip')
570 def _find_filectx(self, title):
571 """Find the last revision in which the file existed."""
573 repo_file = self._title_to_file(title)
574 changectx = self._changectx()
575 stack = [changectx]
576 while repo_file not in changectx:
577 if not stack:
578 return None
579 changectx = stack.pop()
580 for parent in changectx.parents():
581 if parent != changectx:
582 stack.append(parent)
583 return changectx[repo_file]
585 def page_history(self, title):
586 """Iterate over the page's history."""
588 filectx_tip = self._find_filectx(title)
589 if filectx_tip is None:
590 return
591 maxrev = filectx_tip.filerev()
592 minrev = 0
593 for rev in range(maxrev, minrev-1, -1):
594 filectx = filectx_tip.filectx(rev)
595 date = datetime.datetime.fromtimestamp(filectx.date()[0])
596 author = unicode(filectx.user(), "utf-8",
597 'replace').split('<')[0].strip()
598 comment = unicode(filectx.description(), "utf-8", 'replace')
599 yield rev, date, author, comment
601 def page_revision(self, title, rev):
602 """Get binary content of the specified revision of the page."""
604 filectx_tip = self._find_filectx(title)
605 if filectx_tip is None:
606 raise werkzeug.exceptions.NotFound()
607 try:
608 data = filectx_tip.filectx(rev).data()
609 except IndexError:
610 raise werkzeug.exceptions.NotFound()
611 return data
613 def revision_text(self, title, rev):
614 """Get unicode text of the specified revision of the page."""
616 data = self.page_revision(title, rev)
617 text = unicode(data, self.charset, 'replace')
618 return text
620 def history(self):
621 """Iterate over the history of entire wiki."""
623 changectx = self._changectx()
624 maxrev = changectx.rev()
625 minrev = 0
626 for wiki_rev in range(maxrev, minrev-1, -1):
627 change = self.repo.changectx(wiki_rev)
628 date = datetime.datetime.fromtimestamp(change.date()[0])
629 author = unicode(change.user(), "utf-8",
630 'replace').split('<')[0].strip()
631 comment = unicode(change.description(), "utf-8", 'replace')
632 for repo_file in change.files():
633 if repo_file.startswith(self.repo_prefix):
634 title = self._file_to_title(repo_file)
635 try:
636 rev = change[repo_file].filerev()
637 except mercurial.revlog.LookupError:
638 rev = -1
639 yield title, rev, date, author, comment
641 def all_pages(self):
642 """Iterate over the titles of all pages in the wiki."""
644 for filename in os.listdir(self.path):
645 if (os.path.isfile(os.path.join(self.path, filename))
646 and not filename.startswith('.')):
647 yield werkzeug.url_unquote(filename)
649 def changed_since(self, rev):
650 """Return all pages that changed since specified repository revision."""
652 try:
653 last = self.repo.lookup(int(rev))
654 except IndexError:
655 for page in self.all_pages():
656 yield page
657 return
658 current = self.repo.lookup('tip')
659 status = self.repo.status(current, last)
660 modified, added, removed, deleted, unknown, ignored, clean = status
661 for filename in modified+added+removed+deleted:
662 if filename.startswith(self.repo_prefix):
663 yield self._file_to_title(filename)
665 class WikiParser(object):
666 r"""
667 Responsible for generating HTML markup from the wiki markup.
669 The parser works on two levels. On the block level, it analyzes lines
670 of text and decides what kind of block element they belong to (block
671 elements include paragraphs, lists, headings, preformatted blocks).
672 Lines belonging to the same block are joined together, and a second
673 pass is made using regular expressions to parse line-level elements,
674 such as links, bold and italic text and smileys.
676 Some block-level elements, such as preformatted blocks, consume additional
677 lines from the input until they encounter the end-of-block marker, using
678 lines_until. Most block-level elements are just runs of marked up lines
679 though.
682 """
684 bullets_pat = ur"^\s*[*]+\s+"
685 heading_pat = ur"^\s*=+"
686 quote_pat = ur"^[>]+\s+"
687 block = {
688 # "name": (priority, ur"pattern"),
689 "bullets": (10, bullets_pat),
690 "code": (20, ur"^[{][{][{]+\s*$"),
691 "conflict": (30, ur"^<<<<<<< local\s*$"),
692 "empty": (40, ur"^\s*$"),
693 "heading": (50, heading_pat),
694 "indent": (60, ur"^[ \t]+"),
695 "macro":(70, ur"^<<\w+\s*$"),
696 "quote": (80, quote_pat),
697 "rule": (90, ur"^\s*---+\s*$"),
698 "syntax": (100, ur"^\{\{\{\#![\w+#.-]+\s*$"),
699 "table": (110, ur"^\|"),
700 }
701 image_pat = (ur"\{\{(?P<image_target>([^|}]|}[^|}])*)"
702 ur"(\|(?P<image_text>([^}]|}[^}])*))?}}")
703 smilies = {
704 r':)': "smile.png",
705 r':(': "frown.png",
706 r':P': "tongue.png",
707 r':D': "grin.png",
708 r';)': "wink.png",
709 }
710 punct = {
711 r'...': "…",
712 r'--': "–",
713 r'---': "—",
714 r'~': " ",
715 r'\~': "~",
716 r'~~': "∼",
717 r'(C)': "©",
718 r'-->': "→",
719 r'<--': "←",
720 r'(R)': "®",
721 r'(TM)': "™",
722 r'%%': "‰",
723 r'``': "“",
724 r"''": "”",
725 r",,": "„",
726 }
727 markup = {
728 # "name": (priority, ur"pattern"),
729 "bold": (10, ur"[*][*]"),
730 "code": (20, ur"[{][{][{](?P<code_text>([^}]|[^}][}]|[^}][}][}])"
731 ur"*[}]*)[}][}][}]"),
732 "free_link": (30, ur"""(http|https|ftp)://\S+[^\s.,:;!?()'"=+<>-]"""),
733 "italic": (40 , ur"//"),
734 "link": (50, ur"\[\[(?P<link_target>([^|\]]|\][^|\]])+)"
735 ur"(\|(?P<link_text>([^\]]|\][^\]])+))?\]\]"),
736 "image": (60, image_pat),
737 "linebreak": (70, ur"\\\\"),
738 "macro": (80, ur"[<][<](?P<macro_name>\w+)\s+"
739 ur"(?P<macro_text>([^>]|[^>][>])+)[>][>]"),
740 "mail": (90, ur"""(mailto:)?\S+@\S+(\.[^\s.,:;!?()'"/=+<>-]+)+"""),
741 "math": (100, ur"\$\$(?P<math_text>[^$]+)\$\$"),
742 "mono": (110, ur"##"),
743 "newline": (120, ur"\n"),
744 "punct": (130, ur'(^|\b|(?<=\s))(%s)((?=[\s.,:;!?)/&=+])|\b|$)' %
745 ur"|".join(re.escape(k) for k in punct)),
746 "table": (140, ur"=?\|=?"),
747 "text": (150, ur".+?"),
748 }
751 def __init__(self, lines, wiki_link, wiki_image,
752 wiki_syntax=None, wiki_math=None, smilies=None):
753 self.wiki_link = wiki_link
754 self.wiki_image = wiki_image
755 self.wiki_syntax = wiki_syntax
756 self.wiki_math = wiki_math
757 self.enumerated_lines = enumerate(lines)
758 if smilies is not None:
759 self.smilies = smilies
760 self.compile_patterns()
761 self.headings = {}
762 self.stack = []
763 self.line_no = 0
765 def compile_patterns(self):
766 self.quote_re = re.compile(self.quote_pat, re.U)
767 self.heading_re = re.compile(self.heading_pat, re.U)
768 self.bullets_re = re.compile(self.bullets_pat, re.U)
769 patterns = ((k, p) for (k, (x, p)) in
770 sorted(self.block.iteritems(), key=lambda x: x[1][0]))
771 self.block_re = re.compile(ur"|".join("(?P<%s>%s)" % pat
772 for pat in patterns), re.U)
773 self.code_close_re = re.compile(ur"^\}\}\}\s*$", re.U)
774 self.macro_close_re = re.compile(ur"^>>\s*$", re.U)
775 self.conflict_close_re = re.compile(ur"^>>>>>>> other\s*$", re.U)
776 self.conflict_sep_re = re.compile(ur"^=======\s*$", re.U)
777 self.image_re = re.compile(self.image_pat, re.U)
778 smileys = ur"|".join(re.escape(k) for k in self.smilies)
779 smiley_pat = (ur"(^|\b|(?<=\s))(?P<smiley_face>%s)"
780 ur"((?=[\s.,:;!?)/&=+-])|$)" % smileys)
781 self.markup['smiley'] = (125, smiley_pat)
782 patterns = ((k, p) for (k, (x, p)) in
783 sorted(self.markup.iteritems(), key=lambda x: x[1][0]))
784 self.markup_re = re.compile(ur"|".join("(?P<%s>%s)" % pat
785 for pat in patterns), re.U)
787 def __iter__(self):
788 return self.parse()
790 @classmethod
791 def extract_links(cls, text):
792 links = []
793 def link(addr, label=None, class_=None, image=None, alt=None, lineno=0):
794 addr = addr.strip()
795 if external_link(addr):
796 return u''
797 if '#' in addr:
798 addr, chunk = addr.split('#', 1)
799 if addr == u'':
800 return u''
801 links.append((addr, label))
802 return u''
803 lines = text.split('\n')
804 for part in cls(lines, link, link):
805 for ret in links:
806 yield ret
807 links[:] = []
809 def parse(self):
810 """Parse a list of lines of wiki markup, yielding HTML for it."""
812 self.headings = {}
813 self.stack = []
814 self.line_no = 0
816 def key(enumerated_line):
817 line_no, line = enumerated_line
818 match = self.block_re.match(line)
819 if match:
820 return match.lastgroup
821 return "paragraph"
823 for kind, block in itertools.groupby(self.enumerated_lines, key):
824 func = getattr(self, "_block_%s" % kind)
825 for part in func(block):
826 yield part
828 def parse_line(self, line):
829 """
830 Find all the line-level markup and return HTML for it.
832 """
834 for match in self.markup_re.finditer(line):
835 func = getattr(self, "_line_%s" % match.lastgroup)
836 yield func(match.groupdict())
838 def pop_to(self, stop):
839 """
840 Pop from the stack until the specified tag is encoutered.
841 Return string containing closing tags of everything popped.
842 """
843 tags = []
844 tag = None
845 try:
846 while tag != stop:
847 tag = self.stack.pop()
848 tags.append(tag)
849 except IndexError:
850 pass
851 return u"".join(u"</%s>" % tag for tag in tags)
853 def lines_until(self, close_re):
854 """Get lines from input until the closing markup is encountered."""
856 self.line_no, line = self.enumerated_lines.next()
857 while not close_re.match(line):
858 yield line.rstrip()
859 line_no, line = self.enumerated_lines.next()
861 # methods for the markup inside lines:
863 def _line_table(self, groups):
864 return groups["table"]
866 def _line_linebreak(self, groups):
867 return u'<br>'
869 def _line_smiley(self, groups):
870 smiley = groups["smiley_face"]
871 try:
872 url = self.smilies[smiley]
873 except KeyError:
874 url = ''
875 return self.wiki_image(url, smiley, class_="smiley")
877 def _line_bold(self, groups):
878 if 'b' in self.stack:
879 return self.pop_to('b')
880 else:
881 self.stack.append('b')
882 return u"<b>"
884 def _line_italic(self, groups):
885 if 'i' in self.stack:
886 return self.pop_to('i')
887 else:
888 self.stack.append('i')
889 return u"<i>"
891 def _line_mono(self, groups):
892 if 'tt' in self.stack:
893 return self.pop_to('tt')
894 else:
895 self.stack.append('tt')
896 return u"<tt>"
898 def _line_punct(self, groups):
899 text = groups["punct"]
900 return self.punct.get(text, text)
902 def _line_newline(self, groups):
903 return "\n"
905 def _line_text(self, groups):
906 return werkzeug.escape(groups["text"])
908 def _line_math(self, groups):
909 if self.wiki_math:
910 return self.wiki_math(groups["math_text"])
911 else:
912 return "<var>%s</var>" % werkzeug.escape(groups["math_text"])
914 def _line_code(self, groups):
915 return u'<code>%s</code>' % werkzeug.escape(groups["code_text"])
917 def _line_free_link(self, groups):
918 groups['link_target'] = groups['free_link']
919 return self._line_link(groups)
921 def _line_mail(self, groups):
922 addr = groups['mail']
923 groups['link_text'] = addr
924 if not addr.startswith(u'mailto:'):
925 addr = u'mailto:%s' % addr
926 groups['link_target'] = addr
927 return self._line_link(groups)
929 def _line_link(self, groups):
930 target = groups['link_target']
931 text = groups.get('link_text')
932 if not text:
933 text = target
934 if '#' in text:
935 text, chunk = text.split('#', 1)
936 match = self.image_re.match(text)
937 if match:
938 image = self._line_image(match.groupdict())
939 return self.wiki_link(target, text, image=image)
940 return self.wiki_link(target, text)
942 def _line_image(self, groups):
943 target = groups['image_target']
944 alt = groups.get('image_text')
945 if alt is None:
946 alt = target
947 return self.wiki_image(target, alt)
949 def _line_macro(self, groups):
950 name = groups['macro_name']
951 text = groups['macro_text'].strip()
952 return u'<span class="%s">%s</span>' % (
953 werkzeug.escape(name, quote=True),
954 werkzeug.escape(text))
956 # methods for the block (multiline) markup:
958 def _block_code(self, block):
959 for self.line_no, part in block:
960 inside = u"\n".join(self.lines_until(self.code_close_re))
961 yield werkzeug.html.pre(werkzeug.html(inside), class_="code",
962 id="line_%d" % self.line_no)
964 def _block_syntax(self, block):
965 for self.line_no, part in block:
966 syntax = part.lstrip('{#!').strip()
967 inside = u"\n".join(self.lines_until(self.code_close_re))
968 if self.wiki_syntax:
969 return self.wiki_syntax(inside, syntax=syntax,
970 line_no=self.line_no)
971 else:
972 return [werkzeug.html.div(werkzeug.html.pre(
973 werkzeug.html(inside), id="line_%d" % self.line_no),
974 class_="highlight")]
976 def _block_macro(self, block):
977 for self.line_no, part in block:
978 name = part.lstrip('<').strip()
979 inside = u"\n".join(self.lines_until(self.macro_close_re))
980 yield u'<div class="%s">%s</div>' % (
981 werkzeug.escape(name, quote=True),
982 werkzeug.escape(inside))
984 def _block_paragraph(self, block):
985 parts = []
986 first_line = None
987 for self.line_no, part in block:
988 if first_line is None:
989 first_line = self.line_no
990 parts.append(part)
991 text = u"".join(self.parse_line(u"".join(parts)))
992 yield werkzeug.html.p(text, self.pop_to(""), id="line_%d" % first_line)
994 def _block_indent(self, block):
995 parts = []
996 first_line = None
997 for self.line_no, part in block:
998 if first_line is None:
999 first_line = self.line_no
1000 parts.append(part.rstrip())
1001 text = u"\n".join(parts)
1002 yield werkzeug.html.pre(werkzeug.html(text), id="line_%d" % first_line)
1004 def _block_table(self, block):
1005 first_line = None
1006 in_head = False
1007 for self.line_no, line in block:
1008 if first_line is None:
1009 first_line = self.line_no
1010 yield u'<table id="line_%d">' % first_line
1011 table_row = line.strip()
1012 is_header = table_row.startswith('|=') and table_row.endswith('=|')
1013 if not in_head and is_header:
1014 in_head = True
1015 yield '<thead>'
1016 elif in_head and not is_header:
1017 in_head = False
1018 yield '</thead>'
1019 yield '<tr>'
1020 in_cell = False
1021 in_th = False
1023 for part in self.parse_line(table_row):
1024 if part in ('=|', '|', '=|=', '|='):
1025 if in_cell:
1026 if in_th:
1027 yield '</th>'
1028 else:
1029 yield '</td>'
1030 in_cell = False
1031 if part in ('=|=', '|='):
1032 in_th = True
1033 else:
1034 in_th = False
1035 else:
1036 if not in_cell:
1037 if in_th:
1038 yield '<th>'
1039 else:
1040 yield '<td>'
1041 in_cell = True
1042 yield part
1043 if in_cell:
1044 if in_th:
1045 yield '</th>'
1046 else:
1047 yield '</td>'
1048 yield '</tr>'
1049 yield u'</table>'
1051 def _block_empty(self, block):
1052 yield u''
1054 def _block_rule(self, block):
1055 for self.line_no, line in block:
1056 yield werkzeug.html.hr()
1058 def _block_heading(self, block):
1059 for self.line_no, line in block:
1060 level = min(len(self.heading_re.match(line).group(0).strip()), 5)
1061 self.headings[level-1] = self.headings.get(level-1, 0)+1
1062 label = u"-".join(str(self.headings.get(i, 0))
1063 for i in range(level))
1064 yield werkzeug.html.a(name="head-%s" % label)
1065 yield u'<h%d id="line_%d">%s</h%d>' % (level, self.line_no,
1066 werkzeug.escape(line.strip("= \t\n\r\v")), level)
1068 def _block_bullets(self, block):
1069 level = 0
1070 in_ul = False
1071 for self.line_no, line in block:
1072 nest = len(self.bullets_re.match(line).group(0).strip())
1073 while nest > level:
1074 if in_ul:
1075 yield '<li>'
1076 yield '<ul id="line_%d">' % self.line_no
1077 in_ul = True
1078 level += 1
1079 while nest < level:
1080 yield '</li></ul>'
1081 in_ul = False
1082 level -= 1
1083 if nest == level and not in_ul:
1084 yield '</li>'
1085 content = line.lstrip().lstrip('*').strip()
1086 yield '<li>%s%s' % (u"".join(self.parse_line(content)),
1087 self.pop_to(""))
1088 in_ul = False
1089 yield '</li></ul>'*level
1091 def _block_quote(self, block):
1092 level = 0
1093 in_p = False
1094 for self.line_no, line in block:
1095 nest = len(self.quote_re.match(line).group(0).strip())
1096 if nest == level:
1097 yield u'\n'
1098 while nest > level:
1099 if in_p:
1100 yield '%s</p>' % self.pop_to("")
1101 in_p = False
1102 yield '<blockquote>'
1103 level += 1
1104 while nest < level:
1105 if in_p:
1106 yield '%s</p>' % self.pop_to("")
1107 in_p = False
1108 yield '</blockquote>'
1109 level -= 1
1110 content = line.lstrip().lstrip('>').strip()
1111 if not in_p:
1112 yield '<p id="line_%d">' % self.line_no
1113 in_p = True
1114 yield u"".join(self.parse_line(content))
1115 if in_p:
1116 yield '%s</p>' % self.pop_to("")
1117 yield '</blockquote>'*level
1119 def _block_conflict(self, block):
1120 for self.line_no, part in block:
1121 yield u'<div class="conflict">'
1122 local = u"\n".join(self.lines_until(self.conflict_sep_re))
1123 yield werkzeug.html.pre(werkzeug.html(local),
1124 class_="local",
1125 id="line_%d" % self.line_no)
1126 other = u"\n".join(self.lines_until(self.conflict_close_re))
1127 yield werkzeug.html.pre(werkzeug.html(other),
1128 class_="other",
1129 id="line_%d" % self.line_no)
1130 yield u'</div>'
1133 class WikiWikiParser(WikiParser):
1134 """A version of WikiParser that recognizes WikiWord links."""
1136 markup = dict(WikiParser.markup)
1137 camel_link = ur"\w+[%s]\w+" % re.escape(
1138 u''.join(unichr(i) for i in xrange(sys.maxunicode)
1139 if unicodedata.category(unichr(i))=='Lu'))
1140 markup["camel_link"] = (105, camel_link)
1141 markup["camel_nolink"] = (106, ur"[!~](?P<camel_text>%s)" % camel_link)
1143 def _line_camel_link(self, groups):
1144 groups['link_target'] = groups['camel_link']
1145 return self._line_link(groups)
1147 def _line_camel_nolink(self, groups):
1148 return werkzeug.escape(groups["camel_text"])
1151 class WikiSearch(object):
1152 """
1153 Responsible for indexing words and links, for fast searching and
1154 backlinks. Uses a cache directory to store the index files.
1155 """
1157 word_pattern = re.compile(ur"""\w[-~&\w]+\w""", re.UNICODE)
1158 jword_pattern = re.compile(
1159 ur"""[ｦ-ﾟ]+|[ぁ-ん～ー]+|[ァ-ヶ～ー]+|[0-9A-Za-z]+|"""
1160 ur"""[０-９Ａ-Ｚａ-ｚΑ-Ωα-ωА-я]+|"""
1161 ur"""[^- !"#$%&'()*+,./:;<=>?@\[\\\]^_`{|}"""
1162 ur"""‾｡｢｣､･ 、。，．・：；？！゛゜´｀¨"""
1163 ur"""＾￣＿／〜‖｜…‥‘’“”"""
1164 ur"""（）〔〕［］｛｝〈〉《》「」『』【】＋−±×÷"""
1165 ur"""＝≠＜＞≦≧∞∴♂♀°′″℃￥＄¢£"""
1166 ur"""％＃＆＊＠§☆★○●◎◇◆□■△▲▽▼※〒"""
1167 ur"""→←↑↓〓∈∋⊆⊇⊂⊃∪∩∧∨¬⇒⇔∠∃∠⊥"""
1168 ur"""⌒∂∇≡≒≪≫√∽∝∵∫∬ʼn♯♭♪†‡¶◾"""
1169 ur"""─│┌┐┘└├┬┤┴┼"""
1170 ur"""━┃┏┓┛┗┣┫┻╋"""
1171 ur"""┠┯┨┷┿┝┰┥┸╂"""
1172 ur"""ｦ-ﾟぁ-ん～ーァ-ヶ"""
1173 ur"""0-9A-Za-z０-９Ａ-Ｚａ-ｚΑ-Ωα-ωА-я]+""", re.UNICODE)
1174 _con = {}
1176 def __init__(self, cache_path, lang, storage):
1177 self.path = cache_path
1178 self.storage = storage
1179 self.lang = lang
1180 if lang == "ja":
1181 self.split_text = self.split_japanese_text
1182 self.filename = os.path.join(cache_path, 'index.sqlite3')
1183 if not os.path.isdir(self.path):
1184 self.empty = True
1185 os.makedirs(self.path)
1186 elif not os.path.exists(self.filename):
1187 self.empty = True
1188 else:
1189 self.empty = False
1190 con = self.con # sqlite3.connect(self.filename)
1191 self.con.execute('CREATE TABLE IF NOT EXISTS titles '
1192 '(id INTEGER PRIMARY KEY, title VARCHAR);')
1193 self.con.execute('CREATE TABLE IF NOT EXISTS words '
1194 '(word VARCHAR, page INTEGER, count INTEGER);')
1195 self.con.execute('CREATE INDEX IF NOT EXISTS index1 '
1196 'ON words (page);')
1197 self.con.execute('CREATE INDEX IF NOT EXISTS index2 '
1198 'ON words (word);')
1199 self.con.execute('CREATE TABLE IF NOT EXISTS links '
1200 '(src INTEGER, target INTEGER, label VARCHAR, number INTEGER);')
1201 self.con.commit()
1202 self.stop_words_re = re.compile(u'^('+u'|'.join(re.escape(_(
1203 u"""am ii iii per po re a about above
1204 across after afterwards again against all almost alone along already also
1205 although always am among ain amongst amoungst amount an and another any aren
1206 anyhow anyone anything anyway anywhere are around as at back be became because
1207 become becomes becoming been before beforehand behind being below beside
1208 besides between beyond bill both but by can cannot cant con could couldnt
1209 describe detail do done down due during each eg eight either eleven else etc
1210 elsewhere empty enough even ever every everyone everything everywhere except
1211 few fifteen fifty fill find fire first five for former formerly forty found
1212 four from front full further get give go had has hasnt have he hence her here
1213 hereafter hereby herein hereupon hers herself him himself his how however
1214 hundred i ie if in inc indeed interest into is it its itself keep last latter
1215 latterly least isn less made many may me meanwhile might mill mine more
1216 moreover most mostly move much must my myself name namely neither never
1217 nevertheless next nine no nobody none noone nor not nothing now nowhere of off
1218 often on once one only onto or other others otherwise our ours ourselves out
1219 over own per perhaps please pre put rather re same see seem seemed seeming
1220 seems serious several she should show side since sincere six sixty so some
1221 somehow someone something sometime sometimes somewhere still such take ten than
1222 that the their theirs them themselves then thence there thereafter thereby
1223 therefore therein thereupon these they thick thin third this those though three
1224 through throughout thru thus to together too toward towards twelve twenty two
1225 un under ve until up upon us very via was wasn we well were what whatever when
1226 whence whenever where whereafter whereas whereby wherein whereupon wherever
1227 whether which while whither who whoever whole whom whose why will with within
1228 without would yet you your yours yourself yourselves""")).split())
1229 +ur')$|.*\d.*', re.U|re.I|re.X)
1233 @property
1234 def con(self):
1235 """Keep one connection per thread."""
1237 thread_id = thread.get_ident()
1238 try:
1239 return self._con[thread_id]
1240 except KeyError:
1241 connection = sqlite3.connect(self.filename)
1242 connection.isolation_level = None
1243 self._con[thread_id] = connection
1244 return connection
1246 def split_text(self, text, stop=True):
1247 """Splits text into words, removing stop words"""
1249 for match in self.word_pattern.finditer(text):
1250 word = match.group(0)
1251 if not (stop and self.stop_words_re.match(word)):
1252 yield word.lower()
1254 def split_japanese_text(self, text, stop=True):
1255 """Splits text into words, including rules for Japanese"""
1257 for match in self.word_pattern.finditer(text):
1258 word = match.group(0)
1259 got_japanese = False
1260 for m in self.jword_pattern.finditer(word):
1261 w = m.group(0)
1262 got_japanese = True
1263 if not (stop and self.stop_words_re.match(w)):
1264 yield w.lower()
1265 if not (got_japanese or stop and self.stop_words_re.match(word)):
1266 yield word.lower()
1268 def count_words(self, words):
1269 count = {}
1270 for word in words:
1271 count[word] = count.get(word, 0)+1
1272 return count
1274 def title_id(self, title, con):
1275 c = con.execute('SELECT id FROM titles WHERE title=?;', (title,))
1276 idents = c.fetchone()
1277 if idents is None:
1278 con.execute('INSERT INTO titles (title) VALUES (?);', (title,))
1279 c = con.execute('SELECT LAST_INSERT_ROWID();')
1280 idents = c.fetchone()
1281 return idents[0]
1283 def update_words(self, title, text, cursor):
1284 title_id = self.title_id(title, cursor)
1285 words = self.count_words(self.split_text(text))
1286 title_words = self.count_words(self.split_text(title))
1287 for word, count in title_words.iteritems():
1288 words[word] = words.get(word, 0) + count
1289 cursor.execute('DELETE FROM words WHERE page=?;', (title_id,))
1290 for word, count in words.iteritems():
1291 cursor.execute('INSERT INTO words VALUES (?, ?, ?);',
1292 (word, title_id, count))
1294 def update_links(self, title, links_and_labels, cursor):
1295 title_id = self.title_id(title, cursor)
1296 cursor.execute('DELETE FROM links WHERE src=?;', (title_id,))
1297 for number, (link, label) in enumerate(links_and_labels):
1298 cursor.execute('INSERT INTO links VALUES (?, ?, ?, ?);',
1299 (title_id, link, label, number))
1301 def orphaned_pages(self):
1302 """Gives all pages with no links to them."""
1304 con = self.con
1305 try:
1306 sql = ('SELECT title FROM titles '
1307 'WHERE NOT EXISTS '
1308 '(SELECT * FROM links WHERE target=title) '
1309 'ORDER BY title;')
1310 for (title,) in con.execute(sql):
1311 yield unicode(title)
1312 finally:
1313 con.commit()
1315 def wanted_pages(self):
1316 """Gives all pages that are linked to, but don't exist, together with
1317 the number of links."""
1319 con = self.con
1320 try:
1321 sql = ('SELECT COUNT(*), target FROM links '
1322 'WHERE NOT EXISTS '
1323 '(SELECT * FROM titles WHERE target=title) '
1324 'GROUP BY target ORDER BY -COUNT(*);')
1325 for (refs, db_title,) in con.execute(sql):
1326 title = unicode(db_title)
1327 if not external_link(title) and not title.startswith('+'):
1328 yield refs, title
1329 finally:
1330 con.commit()
1333 def page_backlinks(self, title):
1334 """Gives a list of pages linking to specified page."""
1336 con = self.con # sqlite3.connect(self.filename)
1337 try:
1338 sql = ('SELECT DISTINCT(titles.title) '
1339 'FROM links, titles '
1340 'WHERE links.target=? AND titles.id=links.src '
1341 'ORDER BY titles.title;')
1342 for (backlink,) in con.execute(sql, (title,)):
1343 yield unicode(backlink)
1344 finally:
1345 con.commit()
1347 def page_links(self, title):
1348 """Gives a list of links on specified page."""
1350 con = self.con # sqlite3.connect(self.filename)
1351 try:
1352 title_id = self.title_id(title, con)
1353 sql = 'SELECT target FROM links WHERE src=? ORDER BY number;'
1354 for (link,) in con.execute(sql, (title_id,)):
1355 yield unicode(link)
1356 finally:
1357 con.commit()
1359 def page_links_and_labels (self, title):
1360 con = self.con # sqlite3.connect(self.filename)
1361 try:
1362 title_id = self.title_id(title, con)
1363 sql = 'SELECT target, label FROM links WHERE src=? ORDER BY number;'
1364 for link, label in con.execute(sql, (title_id,)):
1365 yield unicode(link), unicode(label)
1366 finally:
1367 con.commit()
1369 def find(self, words):
1370 """Iterator of all pages containing the words, and their scores."""
1372 con = self.con
1373 try:
1374 ranks = []
1375 for word in words:
1376 # Calculate popularity of each word.
1377 sql = 'SELECT SUM(words.count) FROM words WHERE word LIKE ?;'
1378 rank = con.execute(sql, ('%%%s%%' % word,)).fetchone()[0]
1379 # If any rank is 0, there will be no results anyways
1380 if not rank:
1381 return
1382 ranks.append((rank, word))
1383 ranks.sort()
1384 # Start with the least popular word. Get all pages that contain it.
1385 first_rank, first = ranks[0]
1386 rest = ranks[1:]
1387 sql = ('SELECT words.page, titles.title, SUM(words.count) '
1388 'FROM words, titles '
1389 'WHERE word LIKE ? AND titles.id=words.page '
1390 'GROUP BY words.page;')
1391 first_counts = con.execute(sql, ('%%%s%%' % first,))
1392 # Check for the rest of words
1393 for title_id, title, first_count in first_counts:
1394 # Score for the first word
1395 score = float(first_count)/first_rank
1396 for rank, word in rest:
1397 sql = ('SELECT SUM(count) FROM words '
1398 'WHERE page=? AND word LIKE ?;')
1399 count = con.execute(sql,
1400 (title_id, '%%%s%%' % word)).fetchone()[0]
1401 if not count:
1402 # If page misses any of the words, its score is 0
1403 score = 0
1404 break
1405 score += float(count)/rank
1406 if score > 0:
1407 yield int(100*score), unicode(title)
1408 finally:
1409 con.commit()
1411 def reindex_page(self, page, title, cursor, text=None):
1412 """Updates the content of the database, needs locks around."""
1414 mime = self.storage.page_mime(title)
1415 if not mime.startswith('text/'):
1416 self.update_words(title, '', cursor=cursor)
1417 return
1418 if text is None:
1419 try:
1420 text = self.storage.page_text(title)
1421 except werkzeug.exceptions.NotFound:
1422 text = u''
1423 extract_links = getattr(page, 'extract_links', None)
1424 if extract_links is not None:
1425 links = extract_links(text)
1426 self.update_links(title, links, cursor=cursor)
1427 self.update_words(title, text, cursor=cursor)
1429 def update_page(self, page, title, data=None, text=None):
1430 """Updates the index with new page content, for a single page."""
1432 if text is None and data is not None:
1433 text = unicode(data, self.storage.charset, 'replace')
1434 cursor = self.con.cursor()
1435 cursor.execute('BEGIN IMMEDIATE TRANSACTION;')
1436 try:
1437 self.set_last_revision(self.storage.repo_revision())
1438 self.reindex_page(page, title, cursor, text)
1439 cursor.execute('COMMIT TRANSACTION;')
1440 except:
1441 cursor.execute('ROLLBACK;')
1442 raise
1444 def reindex(self, wiki, request, pages):
1445 """Updates specified pages in bulk."""
1447 cursor = self.con.cursor()
1448 cursor.execute('BEGIN IMMEDIATE TRANSACTION;')
1449 try:
1450 for title in pages:
1451 page = wiki.get_page(request, title)
1452 self.reindex_page(page, title, cursor)
1453 cursor.execute('COMMIT TRANSACTION;')
1454 self.empty = False
1455 except:
1456 cursor.execute('ROLLBACK;')
1457 raise
1459 def set_last_revision(self, rev):
1460 """Store the last indexed repository revision."""
1462 # We use % here because the sqlite3's substitiution doesn't work
1463 # We store revision 0 as 1, 1 as 2, etc. because 0 means "no revision"
1464 self.con.execute('PRAGMA USER_VERSION=%d;' % (int(rev+1),))
1466 def get_last_revision(self):
1467 """Retrieve the last indexed repository revision."""
1469 con = self.con
1470 c = con.execute('PRAGMA USER_VERSION;')
1471 rev = c.fetchone()[0]
1472 # -1 means "no revision", 1 means revision 0, 2 means revision 1, etc.
1473 return rev-1
1475 def update(self, wiki, request):
1476 """Reindex al pages that changed since last indexing."""
1478 last_rev = self.get_last_revision()
1479 if last_rev == -1:
1480 changed = self.storage.all_pages()
1481 else:
1482 changed = self.storage.changed_since(last_rev)
1483 self.reindex(wiki, request, changed)
1484 rev = self.storage.repo_revision()
1485 self.set_last_revision(rev)
1487 class WikiResponse(werkzeug.BaseResponse, werkzeug.ETagResponseMixin,
1488 werkzeug.CommonResponseDescriptorsMixin):
1489 """A typical HTTP response class made out of Werkzeug's mixins."""
1491 def make_conditional(self, request):
1492 ret = super(WikiResponse, self).make_conditional(request)
1493 # Remove all headers if it's 304, according to
1494 # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5
1495 if self.status.startswith('304'):
1496 self.response = []
1497 try:
1498 del self.content_type
1499 except AttributeError:
1500 pass
1501 try:
1502 del self.content_length
1503 except AttributeError:
1504 pass
1505 try:
1506 del self.headers['Content-length']
1507 except (KeyError, IndexError):
1508 pass
1509 try:
1510 del self.headers['Content-type']
1511 except (KeyError, IndexError):
1512 pass
1513 return ret
1515 class WikiTempFile(object):
1516 """Wrap a file for uploading content."""
1518 def __init__(self, tmppath):
1519 self.tmppath = tempfile.mkdtemp(dir=tmppath)
1520 self.tmpname = os.path.join(self.tmppath, 'saved')
1521 self.f = open(self.tmpname, "wb")
1523 def read(self, *args, **kw):
1524 return self.f.read(*args, **kw)
1526 def readlines(self, *args, **kw):
1527 return self.f.readlines(*args, **kw)
1529 def write(self, *args, **kw):
1530 return self.f.write(*args, **kw)
1532 def seek(self, *args, **kw):
1533 return self.f.seek(*args, **kw)
1535 def truncate(self, *args, **kw):
1536 return self.f.truncate(*args, **kw)
1538 def close(self, *args, **kw):
1539 ret = self.f.close(*args, **kw)
1540 try:
1541 os.unlink(self.tmpname)
1542 except OSError:
1543 pass
1544 try:
1545 os.rmdir(self.tmppath)
1546 except OSError:
1547 pass
1548 return ret
1551 class WikiRequest(werkzeug.BaseRequest, werkzeug.ETagRequestMixin):
1552 """
1553 A Werkzeug's request with additional functions for handling file
1554 uploads and wiki-specific link generation.
1555 """
1557 charset = 'utf-8'
1558 encoding_errors = 'ignore'
1560 def __init__(self, wiki, adapter, environ, **kw):
1561 werkzeug.BaseRequest.__init__(self, environ, shallow=False, **kw)
1562 self.wiki = wiki
1563 self.adapter = adapter
1564 self.tmpfiles = []
1565 self.tmppath = wiki.path
1566 # Whether to print the css for highlighting
1567 self.print_highlight_styles = True
1569 def get_url(self, title=None, view=None, method='GET',
1570 external=False, **kw):
1571 if view is None:
1572 view = self.wiki.view
1573 if title is not None:
1574 kw['title'] = title.strip()
1575 return self.adapter.build(view, kw, method=method,
1576 force_external=external)
1578 def get_download_url(self, title):
1579 return self.get_url(title, view=self.wiki.download)
1581 def get_author(self):
1582 """Try to guess the author name. Use IP address as last resort."""
1584 try:
1585 cookie = werkzeug.url_unquote(self.cookies.get("author", ""))
1586 except UnicodeError:
1587 cookie = None
1588 try:
1589 auth = werkzeug.url_unquote(self.environ.get('REMOTE_USER', ""))
1590 except UnicodeError:
1591 auth = None
1592 author = (self.form.get("author") or cookie or auth or self.remote_addr)
1593 return author
1595 def _get_file_stream(self, total_content_length=None, content_type=None,
1596 filename=None, content_length=None):
1597 """Save all the POSTs to temporary files."""
1599 temp_file = WikiTempFile(self.tmppath)
1600 self.tmpfiles.append(temp_file)
1601 return temp_file
1603 def cleanup(self):
1604 """Clean up the temporary files created by POSTs."""
1606 for temp_file in self.tmpfiles:
1607 temp_file.close()
1608 self.tmpfiles = []
1610 class WikiPage(object):
1611 """Everything needed for rendering a page."""
1613 def __init__(self, wiki, request, title, mime):
1614 self.request = request
1615 self.title = title
1616 self.mime = mime
1617 # for now we just use the globals from wiki object
1618 self.get_url = self.request.get_url
1619 self.get_download_url = self.request.get_download_url
1620 self.wiki = wiki
1621 self.storage = self.wiki.storage
1622 self.index = self.wiki.index
1623 self.config = self.wiki.config
1625 def date_html(self, datetime):
1626 """
1627 Create HTML for a date, according to recommendation at
1628 http://microformats.org/wiki/date
1629 """
1631 text = datetime.strftime('%Y-%m-%d %H:%M')
1632 # We are going for YYYY-MM-DDTHH:MM:SSZ
1633 title = datetime.strftime('%Y-%m-%dT%H:%M:%SZ')
1634 html = werkzeug.html.abbr(text, class_="date", title=title)
1635 return html
1638 def wiki_link(self, addr, label=None, class_=None, image=None, lineno=0):
1639 """Create HTML for a wiki link."""
1641 addr = addr.strip()
1642 text = werkzeug.escape(label or addr)
1643 chunk = ''
1644 if class_ is not None:
1645 classes = [class_]
1646 else:
1647 classes = []
1648 if external_link(addr):
1649 if addr.startswith('mailto:'):
1650 class_ = 'external email'
1651 text = text.replace('@', '@').replace('.', '.')
1652 href = addr.replace('@', '%40').replace('.', '%2E')
1653 else:
1654 classes.append('external')
1655 href = werkzeug.escape(addr, quote=True)
1656 else:
1657 if '#' in addr:
1658 addr, chunk = addr.split('#', 1)
1659 chunk = '#'+chunk
1660 if addr.startswith('+'):
1661 href = '/'.join([self.request.script_root,
1662 '+'+werkzeug.escape(addr[1:], quote=True)])
1663 classes.append('special')
1664 elif addr == u'':
1665 href = chunk
1666 classes.append('anchor')
1667 else:
1668 classes.append('wiki')
1669 href = self.get_url(addr) + chunk
1670 if addr not in self.storage:
1671 classes.append('nonexistent')
1672 class_ = ' '.join(classes) or None
1673 return werkzeug.html.a(image or text, href=href, class_=class_,
1674 title=addr+chunk)
1676 def wiki_image(self, addr, alt, class_='wiki', lineno=0):
1677 """Create HTML for a wiki image."""
1679 addr = addr.strip()
1680 html = werkzeug.html
1681 chunk = ''
1682 if external_link(addr):
1683 return html.img(src=werkzeug.url_fix(addr), class_="external",
1684 alt=alt)
1685 if '#' in addr:
1686 addr, chunk = addr.split('#', 1)
1687 if addr == '':
1688 return html.a(name=chunk)
1689 if addr in self.storage:
1690 mime = self.storage.page_mime(addr)
1691 if mime.startswith('image/'):
1692 return html.img(src=self.get_download_url(addr), class_=class_,
1693 alt=alt)
1694 else:
1695 return html.img(href=self.get_download_url(addr), alt=alt)
1696 else:
1697 return html.a(html(alt), href=self.get_url(addr))
1699 def search_form(self):
1700 html = werkzeug.html
1701 return html.form(html.div(html.input(name="q", class_="search"),
1702 html.input(class_="button", type_="submit", value=_(u'Search')),
1703 ), method="GET", class_="search",
1704 action=self.get_url(None, self.wiki.search))
1706 def logo(self):
1707 html = werkzeug.html
1708 img = html.img(alt=u"[%s]" % self.wiki.front_page,
1709 src=self.get_download_url(self.wiki.logo_page))
1710 return html.a(img, class_='logo', href=self.get_url(self.wiki.front_page))
1712 def menu(self):
1713 """Generate the menu items"""
1715 html = werkzeug.html
1716 if self.wiki.menu_page in self.storage:
1717 items = self.index.page_links_and_labels(self.wiki.menu_page)
1718 else:
1719 items = [
1720 (self.wiki.front_page, self.wiki.front_page),
1721 ('+history', _(u'Recent changes')),
1722 ]
1723 for link, label in items:
1724 if link == self.title:
1725 class_="current"
1726 else:
1727 class_ = None
1728 yield self.wiki_link(link, label, class_=class_)
1730 def header(self, special_title):
1731 html = werkzeug.html
1732 if self.wiki.logo_page in self.storage:
1733 yield self.logo()
1734 yield self.search_form()
1735 yield html.div(u" ".join(self.menu()), class_="menu")
1736 yield html.h1(html(special_title or self.title))
1738 def html_header(self, special_title, edit_url):
1739 e = lambda x: werkzeug.escape(x, quote=True)
1740 h = werkzeug.html
1741 yield h.title(u'%s - %s' % (e(special_title or self.title),
1742 e(self.wiki.site_name)))
1743 yield h.link(rel="stylesheet", type_="text/css",
1744 href=self.get_url(None, self.wiki.pygments_css))
1745 yield h.link(rel="stylesheet", type_="text/css",
1746 href=self.get_url(None, self.wiki.style_css))
1747 if special_title:
1748 yield h.meta(name="robots", content="NOINDEX,NOFOLLOW")
1749 if edit_url:
1750 yield h.link(rel="alternate", type_="application/wiki",
1751 href=edit_url)
1752 yield h.link(rel="shortcut icon", type_="image/x-icon",
1753 href=self.get_url(None, self.wiki.favicon_ico))
1754 yield h.link(rel="alternate", type_="application/rss+xml",
1755 title=e("%s (RSS)" % self.wiki.site_name),
1756 href=self.get_url(None, self.wiki.rss))
1757 yield h.link(rel="alternate", type_="application/rss+xml",
1758 title=e("%s (ATOM)" % self.wiki.site_name),
1759 href=self.get_url(None, self.wiki.atom))
1760 yield h.script(type_="text/javascript",
1761 src=self.get_url(None, self.wiki.scripts_js))
1763 def footer(self, special_title, edit_url):
1764 if special_title:
1765 footer_links = [
1766 (_(u'Changes'), 'changes',
1767 self.get_url(None, self.wiki.recent_changes)),
1768 (_(u'Index'), 'index',
1769 self.get_url(None, self.wiki.all_pages)),
1770 (_(u'Orphaned'), 'orphaned',
1771 self.get_url(None, self.wiki.orphaned)),
1772 (_(u'Wanted'), 'wanted',
1773 self.get_url(None, self.wiki.wanted)),
1774 ]
1775 else:
1776 footer_links = [
1777 (_(u'Edit'), 'edit', edit_url),
1778 (_(u'History'), 'history',
1779 self.get_url(self.title, self.wiki.history)),
1780 (_(u'Backlinks'), 'backlinks',
1781 self.get_url(self.title, self.wiki.backlinks))
1782 ]
1783 for label, class_, url in footer_links:
1784 if url:
1785 yield werkzeug.html.a(werkzeug.html(label), href=url,
1786 class_=class_)
1787 yield u'\n'
1789 def render_content(self, content, special_title=None):
1790 """The main page template."""
1792 edit_url = None
1793 if not special_title:
1794 try:
1795 self.wiki._check_lock(self.title)
1796 edit_url = self.get_url(self.title, self.wiki.edit)
1797 except werkzeug.exceptions.Forbidden:
1798 pass
1800 yield u"""\
1801 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
1802 "http://www.w3.org/TR/html4/strict.dtd">
1803 <html><head>\n"""
1804 for part in self.html_header(special_title, edit_url):
1805 yield part
1806 yield u'\n</head><body><div class="header">\n'
1807 for part in self.header(special_title):
1808 yield part
1809 yield u'\n</div><div class="content">\n'
1810 for part in content:
1811 yield part
1812 if not special_title or not self.title:
1813 yield u'\n<div class="footer">\n'
1814 for part in self.footer(special_title, edit_url):
1815 yield part
1816 yield u'</div>'
1817 yield u'</div></body></html>'
1820 def pages_list(self, pages, message=None, link=None, _class=None):
1821 """Generate the content of a page list page."""
1823 yield werkzeug.html.p(werkzeug.escape(message) % {'link': link})
1824 yield u'<ul class="%s">' % werkzeug.escape(_class or 'pagelist')
1825 for title in pages:
1826 yield werkzeug.html.li(self.wiki_link(title))
1827 yield u'</ul>'
1829 def history_list(self):
1830 """Generate the content of the history page."""
1832 h = werkzeug.html
1833 max_rev = -1;
1834 title = self.title
1835 link = self.wiki_link(title)
1836 yield h.p(h(_(u'History of changes for %(link)s.')) % {'link': link})
1837 url = self.request.get_url(title, self.wiki.undo, method='POST')
1838 yield u'<form action="%s" method="POST"><ul class="history">' % url
1839 try:
1840 self.wiki._check_lock(title)
1841 read_only = False
1842 except werkzeug.exceptions.Forbidden:
1843 read_only = True
1844 for rev, date, author, comment in self.wiki.storage.page_history(title):
1845 if max_rev < rev:
1846 max_rev = rev
1847 if rev > 0:
1848 date_url = self.request.adapter.build(self.wiki.diff, {
1849 'title': title, 'from_rev': rev-1, 'to_rev': rev})
1850 else:
1851 date_url = self.request.adapter.build(self.wiki.revision, {
1852 'title': title, 'rev': rev})
1853 if read_only:
1854 button = u''
1855 else:
1856 button = h.input(type_="submit", name=str(rev),
1857 value=h(_(u'Undo')))
1858 yield h.li(h.a(self.date_html(date), href=date_url),
1859 button, ' . . . . ', h.i(self.wiki_link(author)),
1860 h.div(h(comment), class_="comment"))
1861 yield u'</ul>'
1862 yield h.input(type_="hidden", name="parent", value=max_rev)
1863 yield u'</form>'
1866 def dependencies(self):
1867 """Refresh the page when any of those pages was changed."""
1869 dependencies = set()
1870 for title in [self.wiki.logo_page, self.wiki.menu_page]:
1871 if title not in self.storage:
1872 dependencies.add(werkzeug.url_quote(title))
1873 for title in [self.wiki.menu_page]:
1874 if title in self.storage:
1875 inode, size, mtime = self.storage.page_file_meta(title)
1876 etag = '%s/%d-%d' % (werkzeug.url_quote(title), inode, mtime)
1877 dependencies.add(etag)
1878 return dependencies
1880 class WikiPageText(WikiPage):
1881 """Pages of mime type text/* use this for display."""
1883 def content_iter(self, lines):
1884 yield '<pre>'
1885 for line in lines:
1886 yield werkzeug.html(line)
1887 yield '</pre>'
1889 def view_content(self, lines=None):
1890 """Read the page content from storage or preview and return iterator."""
1892 if lines is None:
1893 f = self.storage.open_page(self.title)
1894 lines = self.storage.page_lines(f)
1895 return self.content_iter(lines)
1897 def editor_form(self, preview=None):
1898 """Generate the HTML for the editor."""
1900 author = self.request.get_author()
1901 lines = []
1902 try:
1903 page_file = self.storage.open_page(self.title)
1904 lines = self.storage.page_lines(page_file)
1905 (rev, old_date, old_author,
1906 old_comment) = self.storage.page_meta(self.title)
1907 comment = _(u'modified')
1908 if old_author == author:
1909 comment = old_comment
1910 except werkzeug.exceptions.NotFound:
1911 comment = _(u'created')
1912 rev = -1
1913 except werkzeug.exceptions.Forbidden:
1914 yield werkzeug.html.p(werkzeug.html(_(u"Can't edit a symbolic link.")))
1915 return
1916 if preview:
1917 lines = preview
1918 comment = self.request.form.get('comment', comment)
1919 html = werkzeug.html
1920 yield u'<form action="" method="POST" class="editor"><div>'
1921 yield u'<textarea name="text" cols="80" rows="20" id="editortext">'
1922 for line in lines:
1923 yield werkzeug.escape(line)
1924 yield u"""</textarea>"""
1925 yield html.input(type_="hidden", name="parent", value=rev)
1926 yield html.label(html(_(u'Comment')), html.input(name="comment",
1927 value=comment), class_="comment")
1928 yield html.label(html(_(u'Author')), html.input(name="author",
1929 value=self.request.get_author()), class_="comment")
1930 yield html.div(
1931 html.input(type_="submit", name="save", value=_(u'Save')),
1932 html.input(type_="submit", name="preview", value=_(u'Preview')),
1933 html.input(type_="submit", name="cancel", value=_(u'Cancel')),
1934 class_="buttons")
1935 yield u'</div></form>'
1936 if preview:
1937 yield html.h1(html(_(u'Preview, not saved')), class_="preview")
1938 for part in self.view_content(preview):
1939 yield part
1941 def diff_content(self, from_text, to_text, message=u''):
1942 """Generate the HTML markup for a diff."""
1944 def infiniter(iterator):
1945 """Turn an iterator into an infinite one, padding it with None"""
1947 for i in iterator:
1948 yield i
1949 while True:
1950 yield None
1952 diff = difflib._mdiff(from_text.split('\n'), to_text.split('\n'))
1953 stack = []
1954 mark_re = re.compile('\0[-+^]([^\1\0]*)\1|([^\0\1])')
1955 yield message
1956 yield u'<pre class="diff">'
1957 for old_line, new_line, changed in diff:
1958 old_no, old_text = old_line
1959 new_no, new_text = new_line
1960 line_no = (new_no or old_no or 1)-1
1961 if changed:
1962 yield u'<div class="change" id="line_%d">' % line_no
1963 old_iter = infiniter(mark_re.finditer(old_text))
1964 new_iter = infiniter(mark_re.finditer(new_text))
1965 old = old_iter.next()
1966 new = new_iter.next()
1967 buff = u''
1968 while old or new:
1969 while old and old.group(1):
1970 if buff:
1971 yield werkzeug.escape(buff)
1972 buff = u''
1973 yield u'<del>%s</del>' % werkzeug.escape(old.group(1))
1974 old = old_iter.next()
1975 while new and new.group(1):
1976 if buff:
1977 yield werkzeug.escape(buff)
1978 buff = u''
1979 yield u'<ins>%s</ins>' % werkzeug.escape(new.group(1))
1980 new = new_iter.next()
1981 if new:
1982 buff += new.group(2)
1983 old = old_iter.next()
1984 new = new_iter.next()
1985 if buff:
1986 yield werkzeug.escape(buff)
1987 yield u'</div>'
1988 else:
1989 yield u'<div class="orig" id="line_%d">%s</div>' % (
1990 line_no, werkzeug.escape(old_text))
1991 yield u'</pre>'
1993 class WikiPageColorText(WikiPageText):
1994 """Text pages, but displayed colorized with pygments"""
1996 def view_content(self, lines=None):
1997 """Generate HTML for the content."""
1999 if lines is None:
2000 text = self.storage.page_text(self.title)
2001 else:
2002 text = ''.join(lines)
2003 return self.highlight(text, mime=self.mime)
2005 def highlight(self, text, mime=None, syntax=None, line_no=0):
2006 """Colorize the source code."""
2008 if pygments is None:
2009 yield werkzeug.html.pre(werkzeug.html(text))
2010 return
2012 formatter = pygments.formatters.HtmlFormatter()
2013 formatter.line_no = line_no
2015 def wrapper(source, outfile):
2016 """Wrap each line of formatted output."""
2018 yield 0, '<div class="highlight"><pre>'
2019 for lineno, line in source:
2020 yield (lineno,
2021 werkzeug.html.span(line, id_="line_%d" %
2022 formatter.line_no))
2023 formatter.line_no += 1
2024 yield 0, '</pre></div>'
2026 formatter.wrap = wrapper
2027 try:
2028 if mime:
2029 lexer = pygments.lexers.get_lexer_for_mimetype(mime)
2030 elif syntax:
2031 lexer = pygments.lexers.get_lexer_by_name(syntax)
2032 else:
2033 lexer = pygments.lexers.guess_lexer(text)
2034 except pygments.util.ClassNotFound:
2035 yield werkzeug.html.pre(werkzeug.html(text))
2036 return
2037 html = pygments.highlight(text, lexer, formatter)
2038 yield html
2040 class WikiPageWiki(WikiPageColorText):
2041 """Pages of with wiki markup use this for display."""
2043 def __init__(self, *args, **kw):
2044 super(WikiPageWiki, self).__init__(*args, **kw)
2045 if self.config.get_bool('wiki_words', False):
2046 self.parser = WikiWikiParser
2047 else:
2048 self.parser = WikiParser
2049 if self.config.get_bool('ignore_indent', False):
2050 try:
2051 del self.parser.block['indent']
2052 except KeyError:
2053 pass
2055 def extract_links(self, text=None):
2056 """Extract all links from the page."""
2058 if text is None:
2059 try:
2060 text = self.storage.page_text(self.title)
2061 except werkzeug.exceptions.NotFound:
2062 text = u''
2063 return self.parser.extract_links(text)
2065 def view_content(self, lines=None):
2066 if lines is None:
2067 f = self.storage.open_page(self.title)
2068 lines = self.storage.page_lines(f)
2069 if self.wiki.icon_page and self.wiki.icon_page in self.storage:
2070 icons = self.index.page_links_and_labels(self.wiki.icon_page)
2071 smilies = dict((emo, link) for (link, emo) in icons)
2072 else:
2073 smilies = None
2074 content = self.parser(lines, self.wiki_link, self.wiki_image,
2075 self.highlight, self.wiki_math, smilies)
2076 return content
2078 def wiki_math(self, math):
2079 math_url = self.config.get('math_url',
2080 'http://www.mathtran.org/cgi-bin/mathtran?tex=')
2081 if '%s' in math_url:
2082 url = math_url % werkzeug.url_quote(math)
2083 else:
2084 url = '%s%s' % (math_url, werkzeug.url_quote(math))
2085 label = werkzeug.escape(math, quote=True)
2086 return werkzeug.html.img(src=url, alt=label, class_="math")
2088 def dependencies(self):
2089 dependencies = WikiPage.dependencies(self)
2090 for title in [self.wiki.icon_page]:
2091 if title in self.storage:
2092 inode, size, mtime = self.storage.page_file_meta(title)
2093 etag = '%s/%d-%d' % (werkzeug.url_quote(title), inode, mtime)
2094 dependencies.add(etag)
2095 for link in self.index.page_links(self.title):
2096 if link not in self.storage:
2097 dependencies.add(werkzeug.url_quote(link))
2098 return dependencies
2100 class WikiPageFile(WikiPage):
2101 """Pages of all other mime types use this for display."""
2103 def view_content(self, lines=None):
2104 if self.title not in self.storage:
2105 raise werkzeug.exceptions.NotFound()
2106 content = ['<p>Download <a href="%s">%s</a> as <i>%s</i>.</p>' %
2107 (self.request.get_download_url(self.title),
2108 werkzeug.escape(self.title), self.mime)]
2109 return content
2111 def editor_form(self, preview=None):
2112 author = self.request.get_author()
2113 if self.title in self.storage:
2114 comment = _(u'changed')
2115 (rev, old_date, old_author,
2116 old_comment) = self.storage.page_meta(self.title)
2117 if old_author == author:
2118 comment = old_comment
2119 else:
2120 comment = _(u'uploaded')
2121 rev = -1
2122 html = werkzeug.html
2123 yield html.p(html(
2124 _(u"This is a binary file, it can't be edited on a wiki. "
2125 u"Please upload a new version instead.")))
2126 yield html.form(html.div(
2127 html.div(html.input(type_="file", name="data"), class_="upload"),
2128 html.input(type_="hidden", name="parent", value=rev),
2129 html.label(html(_(u'Comment')), html.input(name="comment",
2130 value=comment)),
2131 html.label(html(_(u'Author')), html.input(name="author",
2132 value=author)),
2133 html.div(html.input(type_="submit", name="save", value=_(u'Save')),
2134 html.input(type_="submit", name="cancel",
2135 value=_(u'Cancel')),
2136 class_="buttons")), action="", method="POST", class_="editor",
2137 enctype="multipart/form-data")
2139 class WikiPageImage(WikiPageFile):
2140 """Pages of mime type image/* use this for display."""
2142 render_file = '128x128.png'
2144 def view_content(self, lines=None):
2145 if self.title not in self.storage:
2146 raise werkzeug.exceptions.NotFound()
2147 content = ['<img src="%s" alt="%s">'
2148 % (self.request.get_url(self.title, self.wiki.render),
2149 werkzeug.escape(self.title))]
2150 return content
2152 def render_mime(self):
2153 """Give the filename and mime type of the rendered thumbnail."""
2155 if not Image:
2156 raise NotImplementedError('No Image library available')
2157 return self.render_file, 'image/png'
2159 def render_cache(self, cache_dir):
2160 """Render the thumbnail and save in the cache."""
2162 if not Image:
2163 raise NotImplementedError('No Image library available')
2164 page_file = self.storage.open_page(self.title)
2165 cache_path = os.path.join(cache_dir, self.render_file)
2166 cache_file = open(cache_path, 'wb')
2167 try:
2168 im = Image.open(page_file)
2169 im = im.convert('RGBA')
2170 im.thumbnail((128, 128), Image.ANTIALIAS)
2171 im.save(cache_file,'PNG')
2172 except IOError:
2173 raise werkzeug.exceptions.UnsupportedMediaType('Image corrupted')
2174 cache_file.close()
2175 return cache_path
2177 class WikiPageCSV(WikiPageFile):
2178 """Display class for type text/csv."""
2180 def content_iter(self, lines=None):
2181 import csv
2182 # XXX Add preview support
2183 csv_file = self.storage.open_page(self.title)
2184 reader = csv.reader(csv_file)
2185 html_title = werkzeug.escape(self.title, quote=True)
2186 yield u'<table id="%s" class="csvfile">' % html_title
2187 try:
2188 for row in reader:
2189 yield u'<tr>%s</tr>' % (u''.join(u'<td>%s</td>' % cell
2190 for cell in row))
2191 except csv.Error, e:
2192 yield u'</table>'
2193 yield _(u'<p class="error">Error parsing CSV file %s on line %d: %s'
2194 % (html_title, reader.line_num, e))
2195 finally:
2196 csv_file.close()
2197 yield u'</table>'
2199 def view_content(self, lines=None):
2200 if self.title not in self.storage:
2201 raise werkzeug.exceptions.NotFound()
2202 return self.content_iter(lines)
2204 class WikiPageRST(WikiPageText):
2205 """
2206 Display ReStructured Text.
2207 """
2209 def content_iter(self, lines):
2210 try:
2211 from docutils.core import publish_parts
2212 except ImportError:
2213 return super(WikiPageRST, self).content_iter(lines)
2214 text = ''.join(lines)
2215 SAFE_DOCUTILS = dict(file_insertion_enabled=False, raw_enabled=False)
2216 content = publish_parts(text, writer_name='html',
2217 settings_overrides=SAFE_DOCUTILS)['html_body']
2218 return [content]
2221 class WikiPageBugs(WikiPageText):
2222 """
2223 Display class for type text/x-bugs
2224 Parse the ISSUES file in (roughly) format used by ciss
2225 """
2227 def content_iter(self, lines):
2228 last_lines = []
2229 in_header = False
2230 in_bug = False
2231 attributes = {}
2232 title = None
2233 for line_no, line in enumerate(lines):
2234 if last_lines and line.startswith('----'):
2235 title = ''.join(last_lines)
2236 last_lines = []
2237 in_header = True
2238 attributes = {}
2239 elif in_header and ':' in line:
2240 attribute, value = line.split(':', 1)
2241 attributes[attribute.strip()] = value.strip()
2242 else:
2243 if in_header:
2244 if in_bug:
2245 yield '</div>'
2246 tags = [tag.strip() for tag in
2247 attributes.get('tags', '').split()
2248 if tag.strip()]
2249 yield '<div id="line_%d">' % (line_no)
2250 in_bug = True
2251 if title:
2252 yield werkzeug.html.h2(werkzeug.html(title))
2253 if attributes:
2254 yield '<dl>'
2255 for attribute, value in attributes.iteritems():
2256 yield werkzeug.html.dt(werkzeug.html(attribute))
2257 yield werkzeug.html.dd(werkzeug.html(value))
2258 yield '</dl>'
2259 in_header = False
2260 if not line.strip():
2261 if last_lines:
2262 if last_lines[0][0] in ' \t':
2263 yield werkzeug.html.pre(werkzeug.html(
2264 ''.join(last_lines)))
2265 else:
2266 yield werkzeug.html.p(werkzeug.html(
2267 ''.join(last_lines)))
2268 last_lines = []
2269 else:
2270 last_lines.append(line)
2271 if last_lines:
2272 if last_lines[0][0] in ' \t':
2273 yield werkzeug.html.pre(werkzeug.html(
2274 ''.join(last_lines)))
2275 else:
2276 yield werkzeug.html.p(werkzeug.html(
2277 ''.join(last_lines)))
2278 if in_bug:
2279 yield '</div>'
2282 class WikiTitleConverter(werkzeug.routing.PathConverter):
2283 """Behaves like the path converter, except that it escapes slashes."""
2285 def to_url(self, value):
2286 return werkzeug.url_quote(value.strip(), self.map.charset, safe="")
2288 regex='([^+%]|%[^2]|%2[^Bb]).*'
2290 class WikiAllConverter(werkzeug.routing.BaseConverter):
2291 """Matches everything."""
2293 regex='.*'
2296 class Wiki(object):
2297 """
2298 The main class of the wiki, handling initialization of the whole
2299 application and most of the logic.
2300 """
2301 storage_class = WikiStorage
2302 index_class = WikiSearch
2303 filename_map = {
2304 'README': (WikiPageText, 'text/plain'),
2305 'ISSUES': (WikiPageBugs, 'text/x-bugs'),
2306 'ISSUES.txt': (WikiPageBugs, 'text/x-bugs'),
2307 'COPYING': (WikiPageText, 'text/plain'),
2308 'CHANGES': (WikiPageText, 'text/plain'),
2309 'MANIFEST': (WikiPageText, 'text/plain'),
2310 'favicon.ico': (WikiPageImage, 'image/x-icon'),
2311 }
2312 mime_map = {
2313 'text': WikiPageColorText,
2314 'application/x-javascript': WikiPageColorText,
2315 'application/x-python': WikiPageColorText,
2316 'text/csv': WikiPageCSV,
2317 'text/x-rst': WikiPageRST,
2318 'text/x-wiki': WikiPageWiki,
2319 'image': WikiPageImage,
2320 '': WikiPageFile,
2321 }
2322 icon = base64.b64decode(
2323 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABHNCSVQICAgIfAhki'
2324 'AAAAAlwSFlzAAAEnQAABJ0BfDRroQAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBo'
2325 'AAALWSURBVDiNbdNLaFxlFMDx//fd19x5JdNJm0lIImPaYm2MfSUggrssXBVaChUfi1JwpQtxK7gqu'
2326 'LMbQQQ3bipU0G3Rgg98DBpraWob00kzM6Z5TF7tdObm3vvd46K0TBo/OLtzfnychxIRut+Zo2/19vT'
2327 'kLxXze6biONbGJMRipL39MJyt33rvp+rVT7rzVTfw2vFzLxwcLf/V7oSq1W4hACIkIigUtnaoNecXG'
2328 '2u14T8blQRAd2v7yyN/RLFR6IRM1iedSeFnUvhpDydlI9ow0lcedG3348c1djeQz+WcThjgYZMgGBG'
2329 'SJMEYgzGGODLEoTBYGH4DeHcXoDSSzaRVogQjyaMwhtgYcoUco+Nl5qbnubFw7fr//uB2tXp78uj4c'
2330 '0YJsSTESUxsDCemjjH6YhnbtbA8xaVv7n/0uGZHDx48aH8+17iLJQrf9vCdFL7tkcn7/Pb7r8zdmWP'
2331 '2zqwopa7sAl4/cV4NlvrPbgch7aBN1vUIOw9ZWmmw2dqkb18fQSegOrOgfD9zahfQ37/3su+ljj1T6'
2332 'uCnAyxtoZVGa41tWSilULWfCZdaPD986MsjQxOHdwC9PdmT2tLk0oozpxfYf2SZwp4Iz1X4UZWBe1+'
2333 'z9+5X+OkiruWpYr744ZMmvjn5dvrwoVHLdRzWtobY2Kwx9soyz5ZXuV9fQ5pXCBabXKuXcBwbYwxYe'
2334 'kIppTXAF5VP2xutrVYmm8bzM1z9foSZik1z1SWMNLW1AtMrB/gnnMJxbSxbUV2a/QHQT8Y4c+vvC8V'
2335 'C74VCoZcodvnxux5Msg+THCSKHy2R48YgIb/crITrreZlEYl33MKrYycvvnx88p2BUkkpRyGSEBmDi'
2336 'WI6QcC95UUqM9PBzdqN99fbzc9EJNwBKKUoFw+8NDY8/sFQ/8CE57l5pZRdX6kHqxurW43mv98urM9'
2337 'fjJPouohE8NQ1dkEayAJ5wAe2gRawJSKmO/c/aERMn5m9/ksAAAAASUVORK5CYII=')
2338 scripts = r"""function hatta_dates(){var a=document.getElementsByTagName(
2339 'abbr');var p=function(i){return('00'+i).slice(-2)};for(var i=0;i<a.length;++i)
2340 {var n=a[i];if(n.className==='date'){var m=
2341 /^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z$/.exec(
2342 n.getAttribute('title'));var d=new Date(Date.UTC(+m[1],+m[2]-1,+m[3],+m[4],
2343 +m[5],+m[6]));if(d){var b=-d.getTimezoneOffset()/60;if(b>=0){b="+"+b}
2344 n.textContent=""+d.getFullYear()+"-"+p(d.getMonth()+1)+"-"+p(d.getDate())+" "+
2345 p(d.getHours())+":"+p(d.getMinutes())+" GMT"+b}}}}function hatta_edit(){var b=
2346 document.getElementById('editortext');if(b){var c=0+
2347 document.location.hash.substring(1);var d=b.textContent.match(/(.*\n)/g);var
2348 f='';for(var i=0;i<d.length&&i<c;++i){f+=d[i]}b.focus();if(b.setSelectionRange)
2349 {b.setSelectionRange(f.length,f.length)}else if(b.createTextRange){var g=
2350 b.createTextRange();g.collapse(true);g.moveEnd('character',f.length);
2351 g.moveStart('character',f.length);g.select()}var h=document.createElement('pre'
2352 );b.parentNode.appendChild(h);var k=window.getComputedStyle(b,'');h.style.font=
2353 k.font;h.style.border=k.border;h.style.outline=k.outline;h.style.lineHeight=
2354 k.lineHeight;h.style.letterSpacing=k.letterSpacing;h.style.fontFamily=
2355 k.fontFamily;h.style.fontSize=k.fontSize;h.style.padding=0;h.style.overflow=
2356 'scroll';try{h.style.whiteSpace="-moz-pre-wrap"}catch(e){};try{
2357 h.style.whiteSpace="-o-pre-wrap"}catch(e){};try{h.style.whiteSpace="-pre-wrap"
2358 }catch(e){};try{h.style.whiteSpace="pre-wrap"}catch(e){};h.textContent=f;
2359 b.scrollTop=h.scrollHeight;h.parentNode.removeChild(h)}else{var l='';var m=
2360 document.getElementsByTagName('link');for(var i=0;i<m.length;++i){var n=m[i];
2361 if(n.getAttribute('type')==='application/wiki'){l=n.getAttribute('href')}}if(
2362 l===''){return}var o=['p','h1','h2','h3','h4','h5','h6','pre','ul','div',
2363 'span'];for(var j=0;j<o.length;++j){var m=document.getElementsByTagName(o[j]);
2364 for(var i=0;i<m.length;++i){var n=m[i];if(n.id&&n.id.match(/^line_\d+$/)){
2365 n.ondblclick=function(){var a=l+'#'+this.id.replace('line_','');
2366 document.location.href=a}}}}}}
2367 window.onload=function(){hatta_dates();hatta_edit()}"""
2368 style = """\
2369 html { background: #fff; color: #2e3436;
2370 font-family: sans-serif; font-size: 96% }
2371 body { margin: 1em auto; line-height: 1.3; width: 40em }
2372 a { color: #3465a4; text-decoration: none }
2373 a:hover { text-decoration: underline }
2374 a.wiki:visited { color: #204a87 }
2375 a.nonexistent, a.nonexistent:visited { color: #a40000; }
2376 a.external { color: #3465a4; text-decoration: underline }
2377 a.external:visited { color: #75507b }
2378 a img { border: none }
2379 img.math, img.smiley { vertical-align: middle }
2380 pre { font-size: 100%; white-space: pre-wrap; word-wrap: break-word;
2381 white-space: -moz-pre-wrap; white-space: -pre-wrap;
2382 white-space: -o-pre-wrap; line-height: 1.2; color: #555753 }
2383 div.conflict pre.local { background: #fcaf3e; margin-bottom: 0; color: 000}
2384 div.conflict pre.other { background: #ffdd66; margin-top: 0; color: 000; border-top: #d80 dashed 1px; }
2385 pre.diff div.orig { font-size: 75%; color: #babdb6 }
2386 b.highlight, pre.diff ins { font-weight: bold; background: #fcaf3e;
2387 color: #ce5c00; text-decoration: none }
2388 pre.diff del { background: #eeeeec; color: #888a85; text-decoration: none }
2389 pre.diff div.change { border-left: 2px solid #fcaf3e }
2390 div.footer { border-top: solid 1px #babdb6; text-align: right }
2391 h1, h2, h3, h4 { color: #babdb6; font-weight: normal; letter-spacing: 0.125em}
2392 div.buttons { text-align: center }
2393 input.button, div.buttons input { font-weight: bold; font-size: 100%;
2394 background: #eee; border: solid 1px #babdb6; margin: 0.25em; color: #888a85}
2395 .history input.button { font-size: 75% }
2396 .editor textarea { width: 100%; display: block; font-size: 100%;
2397 border: solid 1px #babdb6; }
2398 .editor label { display:block; text-align: right }
2399 .editor .upload { margin: 2em auto; text-align: center }
2400 form.search input.search, .editor label input { font-size: 100%;
2401 border: solid 1px #babdb6; margin: 0.125em 0 }
2402 .editor label.comment input { width: 32em }
2403 a.logo { float: left; display: block; margin: 0.25em }
2404 div.header h1 { margin: 0; }
2405 div.content { clear: left }
2406 form.search { margin:0; text-align: right; font-size: 80% }
2407 div.snippet { font-size: 80%; color: #888a85 }
2408 div.header div.menu { float: right; margin-top: 1.25em }
2409 div.header div.menu a.current { color: #000 }
2410 hr { background: transparent; border:none; height: 0;
2411 border-bottom: 1px solid #babdb6; clear: both }
2412 blockquote { border-left:.25em solid #ccc; padding-left:.5em; margin-left:0}
2413 abbr.date {border:none}
2414 dt {font-weight: bold; float: left; }
2415 dd {font-style: italic; }
2416 """
2418 def __init__(self, config):
2419 if config.get_bool('show_version', False):
2420 sys.stdout.write("Hatta %s\n" % __version__)
2421 sys.exit()
2422 self.dead = False
2423 self.config = config
2424 self.language = config.get('language', None)
2425 global _
2426 if self.language is not None:
2427 try:
2428 _ = gettext.translation('hatta', 'locale',
2429 languages=[self.language]).ugettext
2430 except IOError:
2431 _ = gettext.translation('hatta', fallback=True,
2432 languages=[self.language]).ugettext
2433 else:
2434 _ = gettext.translation('hatta', fallback=True).ugettext
2435 self.path = os.path.abspath(config.get('pages_path', 'docs'))
2436 self.cache = os.path.abspath(config.get('cache_path', 'cache'))
2437 self.page_charset = config.get('page_charset', 'utf-8')
2438 self.menu_page = self.config.get('menu_page', u'Menu')
2439 self.front_page = self.config.get('front_page', u'Home')
2440 self.logo_page = self.config.get('logo_page', u'logo.png')
2441 self.locked_page = self.config.get('locked_page', u'Locked')
2442 self.site_name = self.config.get('site_name', u'Hatta Wiki')
2443 self.read_only = self.config.get_bool('read_only', False)
2444 self.icon_page = self.config.get('icon_page', None)
2445 self.pygments_style = self.config.get('pygments_style', 'tango')
2447 self.storage = self.storage_class(self.path, self.page_charset)
2448 if not os.path.isdir(self.cache):
2449 os.makedirs(self.cache)
2450 reindex = True
2451 else:
2452 reindex = False
2453 self.index = self.index_class(self.cache, self.language, self.storage)
2454 R = werkzeug.routing.Rule
2455 self.url_map = werkzeug.routing.Map([
2456 R('/', defaults={'title': self.front_page},
2457 endpoint=self.view, methods=['GET', 'HEAD']),
2458 R('/+edit/<title:title>', endpoint=self.edit, methods=['GET']),
2459 R('/+edit/<title:title>', endpoint=self.save, methods=['POST']),
2460 R('/+undo/<title:title>', endpoint=self.undo, methods=['POST']),
2461 R('/+history/', endpoint=self.recent_changes,
2462 methods=['GET', 'HEAD']),
2463 R('/+history/<title:title>/<int:from_rev>:<int:to_rev>',
2464 endpoint=self.diff, methods=['GET', 'HEAD']),
2465 R('/+history/<title:title>/<int:rev>', endpoint=self.revision,
2466 methods=['GET', 'HEAD']),
2467 R('/+history/<title:title>', endpoint=self.history,
2468 methods=['GET', 'HEAD']),
2469 R('/+version/', endpoint=self.version, methods=['GET', 'HEAD']),
2470 R('/+version/<title:title>', endpoint=self.version,
2471 methods=['GET', 'HEAD']),
2472 R('/+download/<title:title>', endpoint=self.download,
2473 methods=['GET', 'HEAD']),
2474 R('/+render/<title:title>', endpoint=self.render,
2475 methods=['GET', 'HEAD']),
2476 R('/<title:title>', endpoint=self.view, methods=['GET', 'HEAD']),
2477 R('/+feed/rss', endpoint=self.rss, methods=['GET', 'HEAD']),
2478 R('/+feed/atom', endpoint=self.atom, methods=['GET', 'HEAD']),
2479 R('/+index', endpoint=self.all_pages, methods=['GET', 'HEAD']),
2480 R('/+orphaned', endpoint=self.orphaned, methods=['GET', 'HEAD']),
2481 R('/+wanted', endpoint=self.wanted, methods=['GET', 'HEAD']),
2482 R('/+search', endpoint=self.search, methods=['GET', 'POST']),
2483 R('/+search/<title:title>', endpoint=self.backlinks,
2484 methods=['GET', 'POST']),
2485 R('/off-with-his-head', endpoint=self.die, methods=['GET']),
2486 R('/+hg<all:path>', endpoint=self.hgweb, strict_slashes=False,
2487 methods=['GET', 'POST', 'HEAD']),
2488 # Pages with default content
2489 R('/favicon.ico', endpoint=self.favicon_ico,
2490 methods=['GET', 'HEAD']),
2491 R('/robots.txt', endpoint=self.robots_txt, methods=['GET', 'HEAD']),
2492 R('/+download/style.css', endpoint=self.style_css,
2493 methods=['GET', 'HEAD']),
2494 R('/+download/pygments.css', endpoint=self.pygments_css,
2495 methods=['GET', 'HEAD']),
2496 R('/+download/scripts.js', endpoint=self.scripts_js,
2497 methods=['GET', 'HEAD']),
2498 ], converters={'title':WikiTitleConverter, 'all':WikiAllConverter})
2500 def get_page(self, request, title):
2501 """Creates a page object based on page's mime type"""
2503 if title:
2504 try:
2505 page_class, mime = self.filename_map[title]
2506 except KeyError:
2507 mime = self.storage.page_mime(title)
2508 major, minor = mime.split('/', 1)
2509 try:
2510 page_class = self.mime_map[mime]
2511 except KeyError:
2512 try:
2513 plus_pos = minor.find('+')
2514 if plus_pos>0:
2515 minor_base = minor[plus_pos:]
2516 else:
2517 minor_base = ''
2518 base_mime = '/'.join([major, minor_base])
2519 page_class = self.mime_map[base_mime]
2520 except KeyError:
2521 try:
2522 page_class = self.mime_map[major]
2523 except KeyError:
2524 page_class = self.mime_map['']
2525 else:
2526 page_class = WikiPage
2527 mime = ''
2528 return page_class(self, request, title, mime)
2530 def view(self, request, title):
2531 page = self.get_page(request, title)
2532 try:
2533 content = page.view_content()
2534 except werkzeug.exceptions.NotFound:
2535 url = request.get_url(title, self.edit, external=True)
2536 return werkzeug.routing.redirect(url, code=303)
2537 html = page.render_content(content)
2538 dependencies = page.dependencies()
2539 etag = '/(%s)' % u','.join(dependencies)
2540 return self.response(request, title, html, etag=etag)
2542 def revision(self, request, title, rev):
2543 text = self.storage.revision_text(title, rev)
2544 link = werkzeug.html.a(werkzeug.html(title),
2545 href=request.get_url(title))
2546 content = [
2547 werkzeug.html.p(
2548 werkzeug.html(
2549 _(u'Content of revision %(rev)d of page %(title)s:'))
2550 % {'rev': rev, 'title': link }),
2551 werkzeug.html.pre(werkzeug.html(text)),
2552 ]
2553 special_title = _(u'Revision of "%(title)s"') % {'title': title}
2554 page = self.get_page(request, title)
2555 html = page.render_content(content, special_title)
2556 response = self.response(request, title, html, rev=rev, etag='/old')
2557 return response
2559 def version(self, request, title=None):
2560 if title is None:
2561 version = self.storage.repo_revision()
2562 else:
2563 try:
2564 version, x, x, x = self.storage.page_history(title).next()
2565 except StopIteration:
2566 version = 0
2567 return werkzeug.Response('%d' % version, mimetype="text/plain")
2569 def _check_lock(self, title):
2570 restricted_pages = [
2571 'scripts.js',
2572 'robots.txt',
2573 ]
2574 if self.read_only:
2575 raise werkzeug.exceptions.Forbidden(_("This site is read-only."))
2576 if title in restricted_pages:
2577 raise werkzeug.exceptions.Forbidden(_("""Can't edit this page.
2578 It can only be edited by the site admin directly on the disk."""))
2579 if title in self.index.page_links(self.locked_page):
2580 raise werkzeug.exceptions.Forbidden(_("This page is locked."))
2582 def save(self, request, title):
2583 self._check_lock(title)
2584 url = request.get_url(title)
2585 if request.form.get('cancel'):
2586 if title not in self.storage:
2587 url = request.get_url(self.front_page)
2588 if request.form.get('preview'):
2589 text = request.form.get("text")
2590 if text is not None:
2591 lines = text.split('\n')
2592 else:
2593 lines = [werkzeug.html.p(werkzeug.html(
2594 _(u'No preview for binaries.')))]
2595 return self.edit(request, title, preview=lines)
2596 elif request.form.get('save'):
2597 comment = request.form.get("comment", "")
2598 author = request.get_author()
2599 text = request.form.get("text")
2600 try:
2601 parent = int(request.form.get("parent"))
2602 except (ValueError, TypeError):
2603 parent = None
2604 self.storage.reopen()
2605 self.index.update(self, request)
2606 page = self.get_page(request, title)
2607 if text is not None:
2608 if title == self.locked_page:
2609 for link, label in page.extract_links(text):
2610 if title == link:
2611 raise werkzeug.exceptions.Forbidden()
2612 if u'href="' in comment or u'http:' in comment:
2613 raise werkzeug.exceptions.Forbidden()
2614 if text.strip() == '':
2615 self.storage.delete_page(title, author, comment)
2616 url = request.get_url(self.front_page)
2617 else:
2618 self.storage.save_text(title, text, author, comment, parent)
2619 else:
2620 text = u''
2621 upload = request.files['data']
2622 f = upload.stream
2623 if f is not None and upload.filename is not None:
2624 try:
2625 self.storage.save_file(title, f.tmpname, author,
2626 comment, parent)
2627 except AttributeError:
2628 self.storage.save_data(title, f.read(), author,
2629 comment, parent)
2630 else:
2631 self.storage.delete_page(title, author, comment)
2632 url = request.get_url(self.front_page)
2633 self.index.update_page(page, title, text=text)
2634 response = werkzeug.routing.redirect(url, code=303)
2635 response.set_cookie('author',
2636 werkzeug.url_quote(request.get_author()),
2637 max_age=604800)
2638 return response
2640 def edit(self, request, title, preview=None):
2641 self._check_lock(title)
2642 exists = title in self.storage
2643 if exists:
2644 self.storage.reopen()
2645 page = self.get_page(request, title)
2646 content = page.editor_form(preview)
2647 special_title = _(u'Editing "%(title)s"') % {'title': title}
2648 html = page.render_content(content, special_title)
2649 if not exists:
2650 response = werkzeug.Response(html, mimetype="text/html",
2651 status='404 Not found')
2653 elif preview:
2654 response = werkzeug.Response(html, mimetype="text/html")
2655 else:
2656 response = self.response(request, title, html, '/edit')
2657 response.headers.add('Cache-Control', 'no-cache')
2658 return response
2660 def atom(self, request):
2661 date_format = "%Y-%m-%dT%H:%M:%SZ"
2662 first_date = datetime.datetime.now()
2663 now = first_date.strftime(date_format)
2664 body = []
2665 first_title = u''
2666 count = 0
2667 unique_titles = {}
2668 for title, rev, date, author, comment in self.storage.history():
2669 if title in unique_titles:
2670 continue
2671 unique_titles[title] = True
2672 count += 1
2673 if count > 10:
2674 break
2675 if not first_title:
2676 first_title = title
2677 first_rev = rev
2678 first_date = date
2679 item = u"""<entry>
2680 <title>%(title)s</title>
2681 <link href="%(page_url)s" />
2682 <content>%(comment)s</content>
2683 <updated>%(date)s</updated>
2684 <author>
2685 <name>%(author)s</name>
2686 <uri>%(author_url)s</uri>
2687 </author>
2688 <id>%(url)s</id>
2689 </entry>""" % {
2690 'title': werkzeug.escape(title),
2691 'page_url': request.adapter.build(self.view, {'title': title},
2692 force_external=True),
2693 'comment': werkzeug.escape(comment),
2694 'date': date.strftime(date_format),
2695 'author': werkzeug.escape(author),
2696 'author_url': request.adapter.build(self.view,
2697 {'title': author},
2698 force_external=True),
2699 'url': request.adapter.build(self.revision,
2700 {'title': title, 'rev': rev},
2701 force_external=True),
2702 }
2703 body.append(item)
2704 content = u"""<?xml version="1.0" encoding="utf-8"?>
2705 <feed xmlns="http://www.w3.org/2005/Atom">
2706 <title>%(title)s</title>
2707 <link rel="self" href="%(atom)s"/>
2708 <link href="%(home)s"/>
2709 <id>%(home)s</id>
2710 <updated>%(date)s</updated>
2711 <logo>%(logo)s</logo>
2712 %(body)s
2713 </feed>""" % {
2714 'title': self.site_name,
2715 'home': request.adapter.build(self.view, force_external=True),
2716 'atom': request.adapter.build(self.atom, force_external=True),
2717 'date': first_date.strftime(date_format),
2718 'logo': request.adapter.build(self.download,
2719 {'title': self.logo_page},
2720 force_external=True),
2721 'body': u''.join(body),
2722 }
2723 response = self.response(request, 'atom', content, '/atom',
2724 'application/xml', first_rev, first_date)
2725 response.set_etag('/atom/%d' % self.storage.repo_revision())
2726 response.make_conditional(request)
2727 return response
2729 def rss(self, request):
2730 """Serve an RSS feed of recent changes."""
2732 first_date = datetime.datetime.now()
2733 now = first_date.strftime("%a, %d %b %Y %H:%M:%S GMT")
2734 rss_body = []
2735 first_title = u''
2736 count = 0
2737 unique_titles = {}
2738 for title, rev, date, author, comment in self.storage.history():
2739 if title in unique_titles:
2740 continue
2741 unique_titles[title] = True
2742 count += 1
2743 if count > 10:
2744 break
2745 if not first_title:
2746 first_title = title
2747 first_rev = rev
2748 first_date = date
2749 item = (u'<item><title>%s</title><link>%s</link>'
2750 u'<description>%s</description><pubDate>%s</pubDate>'
2751 u'<dc:creator>%s</dc:creator><guid>%s</guid></item>' % (
2752 werkzeug.escape(title),
2753 request.adapter.build(self.view, {'title': title},
2754 force_external=True),
2755 werkzeug.escape(comment),
2756 date.strftime("%a, %d %b %Y %H:%M:%S GMT"),
2757 werkzeug.escape(author),
2758 request.adapter.build(self.revision,
2759 {'title': title, 'rev': rev})
2760 ))
2761 rss_body.append(item)
2762 rss_head = u"""<?xml version="1.0" encoding="utf-8"?>
2763 <rss version="2.0"
2764 xmlns:dc="http://purl.org/dc/elements/1.1/"
2765 xmlns:atom="http://www.w3.org/2005/Atom"
2766 >
2767 <channel>
2768 <title>%s</title>
2769 <atom:link href="%s" rel="self" type="application/rss+xml" />
2770 <link>%s</link>
2771 <description>%s</description>
2772 <generator>Hatta Wiki</generator>
2773 <language>en</language>
2774 <lastBuildDate>%s</lastBuildDate>
2776 """ % (
2777 werkzeug.escape(self.site_name),
2778 request.adapter.build(self.rss),
2779 request.adapter.build(self.recent_changes),
2780 werkzeug.escape(_(u'Track the most recent changes to the wiki '
2781 u'in this feed.')),
2782 first_date,
2783 )
2784 content = [rss_head]+rss_body+[u'</channel></rss>']
2785 response = self.response(request, 'rss', content, '/rss',
2786 'application/xml', first_rev, first_date)
2787 response.set_etag('/rss/%d' % self.storage.repo_revision())
2788 response.make_conditional(request)
2789 return response
2791 def response(self, request, title, content, etag='', mime='text/html',
2792 rev=None, date=None, size=None):
2793 """Create a WikiResponse for a page."""
2795 response = WikiResponse(content, mimetype=mime)
2796 if rev is None:
2797 inode, _size, mtime = self.storage.page_file_meta(title)
2798 response.set_etag(u'%s/%s/%d-%d' % (etag, werkzeug.url_quote(title),
2799 inode, mtime))
2800 if size == -1:
2801 size = _size
2802 else:
2803 response.set_etag(u'%s/%s/%s' % (etag, werkzeug.url_quote(title),
2804 rev))
2805 if size:
2806 response.content_length = size
2807 response.make_conditional(request)
2808 return response
2810 def download(self, request, title):
2811 """Serve the raw content of a page."""
2813 mime = self.storage.page_mime(title)
2814 if mime == 'text/x-wiki':
2815 mime = 'text/plain'
2816 f = self.storage.open_page(title)
2817 response = self.response(request, title, f, '/download', mime, size=-1)
2818 return response
2820 def render(self, request, title):
2821 """Serve a thumbnail or otherwise rendered content."""
2823 def file_time_and_size(file_path):
2824 """Get file's modification timestamp and its size."""
2826 try:
2827 (st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size,
2828 st_atime, st_mtime, st_ctime) = os.stat(file_path)
2829 except OSError:
2830 st_mtime = 0
2831 st_size = None
2832 return st_mtime, st_size
2834 def rm_temp_dir(dir_path):
2835 """Delete the directory with subdirectories."""
2837 for root, dirs, files in os.walk(dir_path, topdown=False):
2838 for name in files:
2839 try:
2840 os.remove(os.path.join(root, name))
2841 except OSError:
2842 pass
2843 for name in dirs:
2844 try:
2845 os.rmdir(os.path.join(root, name))
2846 except OSError:
2847 pass
2848 try:
2849 os.rmdir(dir_path)
2850 except OSError:
2851 pass
2853 page = self.get_page(request, title)
2854 try:
2855 cache_filename, cache_mime = page.render_mime()
2856 render = page.render_cache
2857 except (AttributeError, NotImplementedError):
2858 return self.download(request, title)
2860 cache_dir = os.path.join(self.cache, 'render',
2861 werkzeug.url_quote(title, safe=''))
2862 cache_file = os.path.join(cache_dir, cache_filename)
2863 page_inode, page_size, page_mtime = self.storage.page_file_meta(title)
2864 cache_mtime, cache_size = file_time_and_size(cache_file)
2865 if page_mtime > cache_mtime:
2866 if not os.path.exists(cache_dir):
2867 os.makedirs(cache_dir)
2868 try:
2869 temp_dir = tempfile.mkdtemp(dir=cache_dir)
2870 result_file = render(temp_dir)
2871 mercurial.util.rename(result_file, cache_file)
2872 finally:
2873 rm_temp_dir(temp_dir)
2874 f = open(cache_file)
2875 response = self.response(request, title, f, '/render', cache_mime,
2876 size=cache_size)
2877 return response
2879 def undo(self, request, title):
2880 """Revert a change to a page."""
2882 self._check_lock(title)
2883 rev = None
2884 for key in request.form:
2885 try:
2886 rev = int(key)
2887 except ValueError:
2888 pass
2889 author = request.get_author()
2890 if rev is not None:
2891 try:
2892 parent = int(request.form.get("parent"))
2893 except (ValueError, TypeError):
2894 parent = None
2895 self.storage.reopen()
2896 self.index.update(self, request)
2897 if rev == 0:
2898 comment = _(u'Delete page %(title)s') % {'title': title}
2899 data = ''
2900 self.storage.delete_page(title, author, comment)
2901 else:
2902 comment = _(u'Undo of change %(rev)d of page %(title)s') % {
2903 'rev': rev, 'title': title}
2904 data = self.storage.page_revision(title, rev-1)
2905 self.storage.save_data(title, data, author, comment, parent)
2906 page = self.get_page(request, title)
2907 self.index.update_page(page, title, data=data)
2908 url = request.adapter.build(self.history, {'title': title},
2909 method='GET', force_external=True)
2910 return werkzeug.redirect(url, 303)
2912 def history(self, request, title):
2913 """Display history of changes of a page."""
2915 page = self.get_page(request, title)
2916 content = page.render_content(page.history_list(),
2917 _(u'History of "%(title)s"') % {'title': title})
2918 response = self.response(request, title, content, '/history')
2919 return response
2922 def recent_changes(self, request):
2923 """Serve the recent changes page."""
2925 def changes_list(page):
2926 """Generate the content of the recent changes page."""
2928 h = werkzeug.html
2929 yield u'<ul>'
2930 last = {}
2931 lastrev = {}
2932 count = 0
2933 for title, rev, date, author, comment in self.storage.history():
2934 if (author, comment) == last.get(title, (None, None)):
2935 continue
2936 count += 1
2937 if count > 100:
2938 break
2939 if rev > 0:
2940 date_url = request.adapter.build(self.diff, {
2941 'title': title,
2942 'from_rev': rev-1,
2943 'to_rev': lastrev.get(title, rev)
2944 })
2945 elif rev == 0:
2946 date_url = request.adapter.build(self.revision, {
2947 'title': title, 'rev': rev})
2948 else:
2949 date_url = request.adapter.build(self.history, {
2950 'title': title})
2951 last[title] = author, comment
2952 lastrev[title] = rev
2954 yield h.li(h.a(page.date_html(date), href=date_url), ' ',
2955 h.b(page.wiki_link(title)), u' . . . . ',
2956 h.i(page.wiki_link(author)),
2957 h.div(h(comment), class_="comment")
2958 )
2959 yield u'</ul>'
2961 page = self.get_page(request, '')
2962 content = page.render_content(changes_list(page), _(u'Recent changes'))
2963 response = WikiResponse(content, mimetype='text/html')
2964 response.set_etag('/history/%d' % self.storage.repo_revision())
2965 response.make_conditional(request)
2966 return response
2968 def diff(self, request, title, from_rev, to_rev):
2969 """Show the differences between specified revisions."""
2971 page = self.get_page(request, title)
2972 build = request.adapter.build
2973 from_url = build(self.revision, {'title': title, 'rev': from_rev})
2974 to_url = build(self.revision, {'title': title, 'rev': to_rev})
2975 a = werkzeug.html.a
2976 links = {
2977 'link1': a(str(from_rev), href=from_url),
2978 'link2': a(str(to_rev), href=to_url),
2979 'link': a(werkzeug.html(title), href=request.get_url(title)),
2980 }
2981 message = werkzeug.html(_(
2982 u'Differences between revisions %(link1)s and %(link2)s '
2983 u'of page %(link)s.')) % links
2984 diff_content = getattr(page, 'diff_content', None)
2985 if diff_content:
2986 from_text = self.storage.revision_text(page.title, from_rev)
2987 to_text = self.storage.revision_text(page.title, to_rev)
2988 content = page.diff_content(from_text, to_text, message)
2989 else:
2990 content = [werkzeug.html.p(werkzeug.html(
2991 _(u"Diff not available for this kind of pages.")))]
2992 special_title = _(u'Diff for "%(title)s"') % {'title': title}
2993 html = page.render_content(content, special_title)
2994 response = werkzeug.Response(html, mimetype='text/html')
2995 return response
2998 def all_pages(self, request):
2999 """Show index of all pages in the wiki."""
3001 page = self.get_page(request, '')
3002 all_pages = sorted(self.storage.all_pages())
3003 content = page.pages_list(all_pages, _(u'Index of all pages'))
3004 html = page.render_content(content, _(u'Page Index'))
3005 response = WikiResponse(html, mimetype='text/html')
3006 response.set_etag('/+index/%d' % self.storage.repo_revision())
3007 response.make_conditional(request)
3008 return response
3010 def orphaned(self, request):
3011 """Show all pages that don't have backlinks."""
3013 page = self.get_page(request, '')
3014 pages = self.index.orphaned_pages()
3015 content = page.pages_list(pages,
3016 _(u'List of pages with no links to them'))
3017 html = page.render_content(content, _(u'Orphaned pages'))
3018 response = WikiResponse(html, mimetype='text/html')
3019 response.set_etag('/+orphaned/%d' % self.storage.repo_revision())
3020 response.make_conditional(request)
3021 return response
3023 def wanted(self, request):
3024 """Show all pages that don't exist yet, but are linked."""
3026 def wanted_pages_list(page):
3027 """Generate the content of wanted pages page."""
3029 h = werkzeug.html
3030 yield h.p(h(
3031 _(u"List of pages that are linked to, but don't exist yet.")))
3032 yield u'<ol class="wanted">'
3033 for refs, title in self.index.wanted_pages():
3034 url = page.get_url(title, self.backlinks)
3035 yield h.li(h.b(page.wiki_link(title)),
3036 h.i(u' (', h.a(h(_(u"%d references") % refs),
3037 href=url, class_="backlinks"), ')'))
3038 yield u'</ol>'
3040 page = self.get_page(request, '')
3041 content = wanted_pages_list(page)
3042 html = page.render_content(content, _(u'Wanted pages'))
3043 response = WikiResponse(html, mimetype='text/html')
3044 response.set_etag('/+wanted/%d' % self.storage.repo_revision())
3045 response.make_conditional(request)
3046 return response
3048 def search(self, request):
3049 """Serve the search results page."""
3051 def search_snippet(title, words):
3052 """Extract a snippet of text for search results."""
3054 try:
3055 text = self.storage.page_text(title)
3056 except werkzeug.exceptions.NotFound:
3057 return u''
3058 regexp = re.compile(u"|".join(re.escape(w) for w in words),
3059 re.U|re.I)
3060 match = regexp.search(text)
3061 if match is None:
3062 return u""
3063 position = match.start()
3064 min_pos = max(position - 60, 0)
3065 max_pos = min(position + 60, len(text))
3066 snippet = werkzeug.escape(text[min_pos:max_pos])
3067 highlighted = werkzeug.html.b(match.group(0), class_="highlight")
3068 html = regexp.sub(highlighted, snippet)
3069 return html
3071 def page_search(words, page, request):
3072 """Display the search results."""
3074 h = werkzeug.html
3075 self.storage.reopen()
3076 self.index.update(self, request)
3077 result = sorted(self.index.find(words), key=lambda x:-x[0])
3078 yield werkzeug.html.p(h(_(u'%d page(s) containing all words:')
3079 % len(result)))
3080 yield u'<ol class="search">'
3081 for number, (score, title) in enumerate(result):
3082 yield h.li(h.b(page.wiki_link(title)), u' ', h.i(str(score)),
3083 h.div(search_snippet(title, words),
3084 _class="snippet"),
3085 id_="search-%d" % (number+1))
3086 yield u'</ol>'
3088 query = request.values.get('q', u'').strip()
3089 page = self.get_page(request, '')
3090 if not query:
3091 url = request.get_url(view=self.all_pages, external=True)
3092 return werkzeug.routing.redirect(url, code=303)
3093 words = tuple(self.index.split_text(query, stop=False))
3094 if not words:
3095 words = (query,)
3096 title = _(u'Searching for "%s"') % u" ".join(words)
3097 content = page_search(words, page, request)
3098 html = page.render_content(content, title)
3099 return WikiResponse(html, mimetype='text/html')
3101 def backlinks(self, request, title):
3102 """Serve the page with backlinks."""
3104 self.storage.reopen()
3105 self.index.update(self, request)
3106 page = self.get_page(request, title)
3107 message = _(u'Pages that contain a link to %(link)s.')
3108 link = page.wiki_link(title)
3109 pages = self.index.page_backlinks(title)
3110 content = page.pages_list(pages, message, link, _class='backlinks')
3111 html = page.render_content(content, _(u'Links to "%s"') % title)
3112 response = WikiResponse(html, mimetype='text/html')
3113 response.set_etag('/+search/%d' % self.storage.repo_revision())
3114 response.make_conditional(request)
3115 return response
3117 def _serve_default(self, request, title, content, mime):
3118 """Some pages have their default content."""
3120 if title in self.storage:
3121 return self.download(request, title)
3122 response = werkzeug.Response(content, mimetype=mime)
3123 response.set_etag('/%s/-1' % title)
3124 response.make_conditional(request)
3125 return response
3127 def scripts_js(self, request):
3128 """Server the default scripts"""
3130 return self._serve_default(request, 'scripts.js', self.scripts,
3131 'text/javascript')
3133 def style_css(self, request):
3134 """Serve the default style"""
3136 return self._serve_default(request, 'style.css', self.style,
3137 'text/css')
3138 def pygments_css(self, request):
3139 """Serve the default pygments style"""
3141 if pygments is None:
3142 raise werkzeug.exceptions.NotFound()
3144 pygments_style = self.pygments_style
3145 if pygments_style not in pygments.styles.STYLE_MAP:
3146 pygments_style = 'default'
3147 formatter = pygments.formatters.HtmlFormatter(style=pygments_style)
3148 style_defs = formatter.get_style_defs('.highlight')
3149 return self._serve_default(request, 'pygments.css', style_defs,
3150 'text/css')
3152 def favicon_ico(self, request):
3153 """Serve the default favicon."""
3155 return self._serve_default(request, 'favicon.ico', self.icon,
3156 'image/x-icon')
3158 def robots_txt(self, request):
3159 """Serve the robots directives."""
3161 robots = ('User-agent: *\r\n'
3162 'Disallow: /+*\r\n'
3163 'Disallow: /%2B*\r\n'
3164 'Disallow: /+edit\r\n'
3165 'Disallow: /+feed\r\n'
3166 'Disallow: /+history\r\n'
3167 'Disallow: /+search\r\n'
3168 'Disallow: /+hg\r\n'
3169 )
3170 return self._serve_default(request, 'robots.txt', robots,
3171 'text/plain')
3173 def hgweb(self, request, path=None):
3174 """Serve the pages repository on the web like a normal hg repository."""
3176 if not self.config.get_bool('hgweb', False):
3177 raise werkzeug.exceptions.Forbidden('Repository access disabled.')
3178 app = mercurial.hgweb.request.wsgiapplication(
3179 lambda: mercurial.hgweb.hgweb(self.storage.repo, self.site_name))
3180 def hg_app(env, start):
3181 env = request.environ
3182 prefix='/+hg'
3183 if env['PATH_INFO'].startswith(prefix):
3184 env["PATH_INFO"] = env["PATH_INFO"][len(prefix):]
3185 env["SCRIPT_NAME"] += prefix
3186 return app(env, start)
3187 return hg_app
3189 def die(self, request):
3190 """Terminate the standalone server if invoked from localhost."""
3192 if not request.remote_addr.startswith('127.'):
3193 raise werkzeug.exceptions.Forbidden()
3194 def agony():
3195 yield u'Oh dear!'
3196 self.dead = True
3197 return werkzeug.Response(agony(), mimetype='text/plain')
3199 @werkzeug.responder
3200 def application(self, environ, start):
3201 """The main application loop."""
3203 adapter = self.url_map.bind_to_environ(environ)
3204 request = WikiRequest(self, adapter, environ)
3205 try:
3206 try:
3207 endpoint, values = adapter.match()
3208 return endpoint(request, **values)
3209 except werkzeug.exceptions.HTTPException, err:
3210 return err
3211 finally:
3212 request.cleanup()
3213 del request
3214 del adapter
3216 def read_config():
3217 """Read and parse the config."""
3219 config = WikiConfig(
3220 # Here you can modify the configuration: uncomment and change the ones
3221 # you need. Note that it's better use environment variables or command
3222 # line switches.
3224 # interface='',
3225 # port=8080,
3226 # pages_path = 'docs',
3227 # cache_path = 'cache',
3228 # front_page = 'Home',
3229 # site_name = 'Hatta Wiki',
3230 # page_charset = 'UTF-8',
3231 )
3232 config.parse_args()
3233 config.parse_files()
3234 # config.sanitize()
3235 return config
3237 def application(env, start):
3238 """Detect that we are being run as WSGI application."""
3240 global application
3241 config = read_config()
3242 script_dir = os.path.dirname(os.path.abspath(__file__))
3243 if config.get('pages_path') is None:
3244 config.set('pages_path', os.path.join(script_dir, 'docs'))
3245 if config.get('cache_path') is None:
3246 config.set('cache_path', os.path.join(script_dir, 'cache'))
3247 wiki = Wiki(config)
3248 application = wiki.application
3249 return application(env, start)
3251 def main(config=None, wiki=None):
3252 """Start a standalone WSGI server."""
3254 config = config or read_config()
3255 wiki = wiki or Wiki(config)
3256 app = wiki.application
3258 host, port = (config.get('interface', '0.0.0.0'),
3259 int(config.get('port', 8080)))
3260 try:
3261 from cherrypy import wsgiserver
3262 except ImportError:
3263 try:
3264 from cherrypy import _cpwsgiserver as wsgiserver
3265 except ImportError:
3266 import wsgiref.simple_server
3267 server = wsgiref.simple_server.make_server(host, port, app)
3268 try:
3269 server.serve_forever()
3270 except KeyboardInterrupt:
3271 pass
3272 return
3273 apps = [('', app)]
3274 name = wiki.site_name
3275 server = wsgiserver.CherryPyWSGIServer((host, port), apps, server_name=name)
3276 try:
3277 server.start()
3278 except KeyboardInterrupt:
3279 server.stop()
3281 if __name__ == "__main__":
3282 main()