# -*- coding: iso-8859-1 -*-
"""
    MoinMoin - Data associated with a single Request

    @copyright: 2001-2003 by Jürgen Hermann <jh@web.de>
    @copyright: 2003-2006 by Thomas Waldmann
    @license: GNU GPL, see COPYING for details.
"""

import os, re, time, sys, cgi, StringIO
import copy
from MoinMoin import config, wikiutil, user, caching
from MoinMoin import error, multiconfig
from MoinMoin.util import MoinMoinNoFooter, IsWin9x

# umask setting --------------------------------------------------------
def set_umask(new_mask=0777^config.umask):
    """ Set the OS umask value (and ignore potential failures on OSes where
        this is not supported).
        Default: the bitwise inverted value of config.umask
    """
    try:
        old_mask = os.umask(new_mask)
    except:
        # maybe we are on win32?
        pass

# We do this at least once per Python process, when request is imported.
# If other software parts (like twistd's daemonize() function) set an
# unwanted umask, we have to call this again to set the correct one:
set_umask()

# Timing ---------------------------------------------------------------

class Clock:
    """ Helper class for code profiling
        we do not use time.clock() as this does not work across threads
    """

    def __init__(self):
        self.timings = {}
        self.states = {}

    def start(self, timer):
        state = self.states.setdefault(timer, 'new')
        if state == 'new':
            self.timings[timer] = time.time()
            self.states[timer] = 'running'
        elif state == 'running':
            pass # this timer is already running, do nothing
        elif state == 'stopped':
            # if a timer is stopped, timings has the sum of all times it was running
            self.timings[timer] = time.time() - self.timings[timer]
            self.states[timer] = 'running'

    def stop(self, timer):
        state = self.states.setdefault(timer, 'neverstarted')
        if state == 'running':
            self.timings[timer] = time.time() - self.timings[timer]
            self.states[timer] = 'stopped'
        elif state == 'stopped':
            pass # this timer already has been stopped, do nothing
        elif state == 'neverstarted':
            pass # this timer never has been started, do nothing

    def value(self, timer):
        state = self.states.setdefault(timer, 'nosuchtimer')
        if state == 'stopped':
            result = "%.3fs" % self.timings[timer]
        elif state == 'running':
            result = "%.3fs (still running)" % (time.time() - self.timings[timer])
        else:
            result = "- (%s)" % state
        return result

    def dump(self):
        outlist = []
        for timer in self.timings.keys():
            value = self.value(timer)
            outlist.append("%s = %s" % (timer, value))
        outlist.sort()
        return outlist


# Utilities

def cgiMetaVariable(header, scheme='http'):
    """ Return CGI meta variable for header name
    
    e.g 'User-Agent' -> 'HTTP_USER_AGENT'    
    See http://www.faqs.org/rfcs/rfc3875.html section 4.1.18
    """
    var = '%s_%s' % (scheme, header)
    return var.upper().replace('-', '_')
    

# Request Base ----------------------------------------------------------

class RequestBase(object):
    """ A collection for all data associated with ONE request. """

    # Defaults (used by sub classes)
    http_accept_language = 'en'
    server_name = 'localhost'
    server_port = '80'

    # Extra headers we support. Both standalone and twisted store
    # headers as lowercase.
    moin_location = 'x-moin-location'
    proxy_host = 'x-forwarded-host'
    
    def __init__(self, properties={}):

        # twistd's daemonize() overrides our umask, so we reset it here every
        # request. we do it for all request types to avoid similar problems.
        set_umask()

        # Decode values collected by sub classes
        self.path_info = self.decodePagename(self.path_info)

        self.opened_logs = 0
        self.failed = 0
        self._available_actions = None
        self._known_actions = None

        # Pages meta data that we collect in one request
        self.pages = {}
              
        self.sent_headers = 0
        self.user_headers = []
        self.cacheable = 0 # may this output get cached by http proxies/caches?
        self.http_caching_disabled = 0 # see disableHttpCaching()
        self.page = None
        self._dicts = None
        
        # Fix dircaching problems on Windows 9x
        if IsWin9x():
            import dircache
            dircache.reset()

        # Check for dumb proxy requests
        # TODO relying on request_uri will not work on all servers, especially
        # not on external non-Apache servers
        self.forbidden = False
        if self.request_uri.startswith('http://'):
            self.makeForbidden403()

        # Init
        else:
            self.writestack = []
            self.clock = Clock()
            self.clock.start('total')
            # order is important here!
            self.__dict__.update(properties)
            try:
                self._load_multi_cfg()
            except error.NoConfigMatchedError:
                self.makeForbidden(404, 'No wiki configuration matching the URL found!\r\n')
                return

            self.open_logs()

            self.isSpiderAgent = self.check_spider()
        
            # Set decode charsets.  Input from the user is always in
            # config.charset, which is the page charsets. Except
            # path_info, which may use utf-8, and handled by decodePagename.
            self.decode_charsets = [config.charset]
            
            # hierarchical wiki - set rootpage
            from MoinMoin.Page import Page
            #path = self.getPathinfo()
            #if path.startswith('/'):
            #    pages = path[1:].split('/')
            #    if 0: # len(path) > 1:
            #        ## breaks MainPage/SubPage on flat storage
            #        rootname = u'/'.join(pages[:-1])
            #    else:
            #        # this is the usual case, as it ever was...
            #        rootname = u""
            #else:
            #    # no extra path after script name
            #    rootname = u""

            self.args = {}
            self.form = {}

            if not self.query_string.startswith('action=xmlrpc'):
                self.args = self.form = self.setup_args()

            rootname = u''
            self.rootpage = Page(self, rootname, is_rootpage=1)

            self.user = self.get_user_from_form()
            
            if not self.query_string.startswith('action=xmlrpc'):
                if not self.forbidden and self.isForbidden():
                    self.makeForbidden403()
                if not self.forbidden and self.surge_protect():
                    self.makeUnavailable503()

            self.logger = None
            self.pragma = {}
            self.mode_getpagelinks = 0 # is > 0 as long as we are in a getPageLinks call
            self.parsePageLinks_running = {} # avoid infinite recursion by remembering what we are already running

            from MoinMoin import i18n
            self.i18n = i18n
            self.lang = i18n.requestLanguage(self) 
            # Language for content. Page content should use the wiki default lang,
            # but generated content like search results should use the user language.
            self.content_lang = self.cfg.language_default
            self.getText = lambda text, i18n=self.i18n, request=self, lang=self.lang, **kv: i18n.getText(text, request, lang, kv.get('formatted', True))

            self.reset()
        
    def surge_protect(self, kick_him=False):
        """ check if someone requesting too much from us,
            if kick_him is True, we unconditionally blacklist the current user/ip
        """
        limits = self.cfg.surge_action_limits
        if not limits:
            return False
        validuser = self.user.valid
        current_id = validuser and self.user.name or self.remote_addr
        if not validuser and current_id.startswith('127.'): # localnet
            return False
        current_action = self.form.get('action', ['show'])[0]
        
        default_limit = self.cfg.surge_action_limits.get('default', (30, 60))
        
        now = int(time.time())
        surgedict = {}
        surge_detected = False
        
        try:
            cache = caching.CacheEntry(self, 'surgeprotect', 'surge-log')
            if cache.exists():
                data = cache.content(decode=True)
                data = data.split("\n")
                for line in data:
                    try:
                        id, t, action, surge_indicator = line.split("\t")
                        t = int(t)
                        maxnum, dt = limits.get(action, default_limit)
                        if t >= now - dt:
                            events = surgedict.setdefault(id, copy.copy({}))
                            timestamps = events.setdefault(action, copy.copy([]))
                            timestamps.append((t, surge_indicator))
                    except StandardError, err:
                        pass
                
            maxnum, dt = limits.get(current_action, default_limit)
            events = surgedict.setdefault(current_id, copy.copy({}))
            timestamps = events.setdefault(current_action, copy.copy([]))
            surge_detected = len(timestamps) > maxnum

            surge_indicator = surge_detected and "!" or ""
            timestamps.append((now, surge_indicator))
            if surge_detected:
                if len(timestamps) < maxnum * 2:
                    timestamps.append((now + self.cfg.surge_lockout_time, surge_indicator)) # continue like that and get locked out
        
            if current_action != 'AttachFile': # don't add AttachFile accesses to all or picture galleries will trigger SP
                current_action = 'all' # put a total limit on user's requests
                maxnum, dt = limits.get(current_action, default_limit)
                events = surgedict.setdefault(current_id, copy.copy({}))
                timestamps = events.setdefault(current_action, copy.copy([]))

                if kick_him: # ban this guy, NOW
                    timestamps.extend([(now + self.cfg.surge_lockout_time, "!")] * (2*maxnum))

                surge_detected = surge_detected or len(timestamps) > maxnum
            
                surge_indicator = surge_detected and "!" or ""
                timestamps.append((now, surge_indicator))
                if surge_detected:
                    if len(timestamps) < maxnum * 2:
                        timestamps.append((now + self.cfg.surge_lockout_time, surge_indicator)) # continue like that and get locked out
        
            data = []
            for id, events in surgedict.items():
                for action, timestamps in events.items():
                    for t, surge_indicator in timestamps:
                        data.append("%s\t%d\t%s\t%s" % (id, t, action, surge_indicator))
            data = "\n".join(data)
            cache.update(data, encode=True)
        except StandardError, err:
            self.log('%s: %s' % (err.__class__.__name__, str(err)))

        return surge_detected   
        
    def getDicts(self):
        """ Lazy initialize the dicts on the first access """
        if self._dicts is None:
            from MoinMoin import wikidicts
            dicts = wikidicts.GroupDict(self)
            dicts.scandicts()
            self._dicts = dicts
        return self._dicts
        
    def delDicts(self):
        """ Delete the dicts, used by some tests """
        del self._dicts
        self._dicts = None

    dicts = property(getDicts, None, delDicts)
  
    def _load_multi_cfg(self):
        # protect against calling multiple times
        if not hasattr(self, 'cfg'):
            self.clock.start('load_multi_cfg')
            self.cfg = multiconfig.getConfig(self.url)
            self.clock.stop('load_multi_cfg')
            
    def setAcceptedCharsets(self, accept_charset):
        """ Set accepted_charsets by parsing accept-charset header

        Set self.accepted_charsets to an ordered list based on http_accept_charset. 
        
        Reference: http://www.w3.org/Protocols/rfc2616/rfc2616.txt

        TODO: currently no code use this value.

        @param accept_charset: accept-charset header
        """        
        charsets = []
        if accept_charset:
            accept_charset = accept_charset.lower()
            # Add iso-8859-1 if needed
            if (not '*' in accept_charset and
                accept_charset.find('iso-8859-1') < 0):
                accept_charset += ',iso-8859-1'

            # Make a list, sorted by quality value, using Schwartzian Transform
            # Create list of tuples (value, name) , sort, extract names  
            for item in accept_charset.split(','):
                if ';' in item:
                    name, qval = item.split(';')
                    qval = 1.0 - float(qval.split('=')[1])
                else:
                    name, qval = item, 0
                charsets.append((qval, name))                 
            charsets.sort()
            # Remove *, its not clear what we should do with it later
            charsets = [name for qval, name in charsets if name != '*']

        self.accepted_charsets = charsets
          
    def _setup_vars_from_std_env(self, env):
        """ Set common request variables from CGI environment
        
        Parse a standard CGI environment as created by common web servers.
        Reference: http://www.faqs.org/rfcs/rfc3875.html

        @param env: dict like object containing cgi meta variables
        """
        # Values we can just copy
        self.env = env
        self.http_accept_language = env.get('HTTP_ACCEPT_LANGUAGE', self.http_accept_language)
        self.server_name = env.get('SERVER_NAME', self.server_name)
        self.server_port = env.get('SERVER_PORT', self.server_port)
        self.saved_cookie = env.get('HTTP_COOKIE', '')
        self.script_name = env.get('SCRIPT_NAME', '')
        self.path_info = env.get('PATH_INFO', '')
        self.query_string = env.get('QUERY_STRING', '')
        self.request_method = env.get('REQUEST_METHOD', None)
        self.remote_addr = env.get('REMOTE_ADDR', '')
        self.http_user_agent = env.get('HTTP_USER_AGENT', '')
        self.if_modified_since = env.get('If-modified-since') or env.get(cgiMetaVariable('If-modified-since'))
        self.if_none_match = env.get('If-none-match') or env.get(cgiMetaVariable('If-none-match'))

        # REQUEST_URI is not part of CGI spec, but an addition of Apache.
        self.request_uri = env.get('REQUEST_URI', '')
        
        # Values that need more work
        self.setHttpReferer(env.get('HTTP_REFERER'))
        self.setIsSSL(env)
        self.setHost(env.get('HTTP_HOST'))
        self.fixURI(env)
        self.setURL(env)
        
        ##self.debugEnvironment(env)

    def setHttpReferer(self, referer):
        """ Set http_referer, making sure its ascii
        
        IE might send non-ascii value.
        """
        value = ''
        if referer:
            value = unicode(referer, 'ascii', 'replace')
            value = value.encode('ascii', 'replace')
        self.http_referer = value

    def setIsSSL(self, env):
        """ Set is_ssl 
        
        @param env: dict like object containing cgi meta variables
        """
        self.is_ssl = bool(env.get('SSL_PROTOCOL') or
                           env.get('SSL_PROTOCOL_VERSION') or
                           env.get('HTTPS') == 'on')

    def setHost(self, host=None):
        """ Set http_host 
        
        Create from server name and port if missing. Previous code
        default to localhost.
        """
        if not host:
            port = ''
            standardPort = ('80', '443')[self.is_ssl]
            if self.server_port != standardPort:
                port = ':' + self.server_port
            host = self.server_name + port
        self.http_host = host
        
    def fixURI(self, env):
        """ Fix problems with script_name and path_info
        
        Handle the strange charset semantics on Windows and other non
        posix systems. path_info is transformed into the system code
        page by the web server. Additionally, paths containing dots let
        most webservers choke.
        
        Broken environment variables in different environments:
                path_info script_name
        Apache1     X          X      PI does not contain dots
        Apache2     X          X      PI is not encoded correctly
        IIS         X          X      path_info include script_name
        Other       ?          -      ? := Possible and even RFC-compatible.
                                      - := Hopefully not.

        @param env: dict like object containing cgi meta variables
        """ 
        # Fix the script_name when using Apache on Windows.
        server_software = env.get('SERVER_SOFTWARE', '')
        if os.name == 'nt' and server_software.find('Apache/') != -1:
            # Removes elements ending in '.' from the path.
            self.script_name = '/'.join([x for x in self.script_name.split('/') 
                                         if not x.endswith('.')])

        # Fix path_info
        if os.name != 'posix' and self.request_uri != '':
            # Try to recreate path_info from request_uri.
            import urlparse
            scriptAndPath = urlparse.urlparse(self.request_uri)[2]
            path = scriptAndPath.replace(self.script_name, '', 1)            
            self.path_info = wikiutil.url_unquote(path, want_unicode=False)
        elif os.name == 'nt':
            # Recode path_info to utf-8
            path = wikiutil.decodeWindowsPath(self.path_info)
            self.path_info = path.encode("utf-8")
            
            # Fix bug in IIS/4.0 when path_info contain script_name
            if self.path_info.startswith(self.script_name):
                self.path_info = self.path_info[len(self.script_name):]

    def setURL(self, env):
        """ Set url, used to locate wiki config 
        
        This is the place to manipulate url parts as needed.
        
        @param env: dict like object containing cgi meta variables or http headers.
        """
        # If we serve on localhost:8000 and use a proxy on
        # example.com/wiki, our urls will be example.com/wiki/pagename
        # Same for the wiki config - they must use the proxy url.
        self.rewriteHost(env)
        self.rewriteURI(env)
        
        if not self.request_uri:
            self.request_uri = self.makeURI()
        self.url = self.http_host + self.request_uri

    def rewriteHost(self, env):
        """ Rewrite http_host transparently
        
        Get the proxy host using 'X-Forwarded-Host' header, added by
        Apache 2 and other proxy software.
        
        TODO: Will not work for Apache 1 or others that don't add this header.
        
        TODO: If we want to add an option to disable this feature it
        should be in the server script, because the config is not
        loaded at this point, and must be loaded after url is set.
        
        @param env: dict like object containing cgi meta variables or http headers.
        """
        proxy_host = (env.get(self.proxy_host) or
                      env.get(cgiMetaVariable(self.proxy_host)))
        if proxy_host:
            self.http_host = proxy_host

    def rewriteURI(self, env):
        """ Rewrite request_uri, script_name and path_info transparently
        
        Useful when running mod python or when running behind a proxy,
        e.g run on localhost:8000/ and serve as example.com/wiki/.

        Uses private 'X-Moin-Location' header to set the script name.
        This allow setting the script name when using Apache 2
        <location> directive::

            <Location /my/wiki/>
                RequestHeader set X-Moin-Location /my/wiki/
            </location>
        
        TODO: does not work for Apache 1 and others that do not allow
        setting custom headers per request.
        
        @param env: dict like object containing cgi meta variables or http headers.
        """
        location = (env.get(self.moin_location) or 
                    env.get(cgiMetaVariable(self.moin_location)))
        if location is None:
            return
        
        scriptAndPath = self.script_name + self.path_info
        location = location.rstrip('/')
        self.script_name = location
        
        # This may happen when using mod_python
        if scriptAndPath.startswith(location):
            self.path_info = scriptAndPath[len(location):]

        # Recreate the URI from the modified parts
        if self.request_uri:
            self.request_uri = self.makeURI()

    def makeURI(self):
        """ Return uri created from uri parts """
        uri = self.script_name + wikiutil.url_quote(self.path_info)
        if self.query_string:
            uri += '?' + self.query_string
        return uri

    def splitURI(self, uri):
        """ Return path and query splited from uri
        
        Just like CGI environment, the path is unquoted, the query is not.
        """
        if '?' in uri:
            path, query = uri.split('?', 1)
        else:
            path, query = uri, ''
        return wikiutil.url_unquote(path, want_unicode=False), query        

    def get_user_from_form(self):
        """ read the maybe present UserPreferences form and call get_user with the values """
        name = self.form.get('name', [None])[0]
        password = self.form.get('password', [None])[0]
        login = self.form.has_key('login')
        logout = self.form.has_key('logout')
        u = self.get_user_default_unknown(name=name, password=password,
                                          login=login, logout=logout,
                                          user_obj=None)
        return u
    
    def get_user_default_unknown(self, **kw):
        """ call do_auth and if it doesnt return a user object, make some "Unknown User" """
        user_obj = self.get_user_default_None(**kw)
        if user_obj is None:
            user_obj = user.User(self, auth_method="request:427")
        return user_obj

    def get_user_default_None(self, **kw):
        """ loop over auth handlers, return a user obj or None """
        name = kw.get('name')
        password = kw.get('password')
        login = kw.get('login')
        logout = kw.get('logout')
        user_obj = kw.get('user_obj')
        for auth in self.cfg.auth:
            user_obj, continue_flag = auth(self, name=name, password=password,
                                           login=login, logout=logout, user_obj=user_obj)
            if not continue_flag:
                break
        return user_obj
        
    def reset(self):
        """ Reset request state.

        Called after saving a page, before serving the updated
        page. Solves some practical problems with request state
        modified during saving.

        """
        # This is the content language and has nothing to do with
        # The user interface language. The content language can change
        # during the rendering of a page by lang macros
        self.current_lang = self.cfg.language_default

        self._all_pages = None
        # caches unique ids
        self._page_ids = {}
        # keeps track of pagename/heading combinations
        # parsers should use this dict and not a local one, so that
        # macros like TableOfContents in combination with Include can work
        self._page_headings = {}

        if hasattr(self, "_fmt_hd_counters"):
            del self._fmt_hd_counters

    def loadTheme(self, theme_name):
        """ Load the Theme to use for this request.

        @param theme_name: the name of the theme
        @type theme_name: str
        @rtype: int
        @return: success code
                 0 on success
                 1 if user theme could not be loaded,
                 2 if a hard fallback to modern theme was required.
        """
        fallback = 0
        if theme_name == "<default>":
            theme_name = self.cfg.theme_default
        
        try:
            Theme = wikiutil.importPlugin(self.cfg, 'theme', theme_name, 'Theme')
        except wikiutil.PluginMissingError:
            fallback = 1
            try:
                Theme = wikiutil.importPlugin(self.cfg, 'theme', self.cfg.theme_default, 'Theme')
            except wikiutil.PluginMissingError:
                fallback = 2
                from MoinMoin.theme.modern import Theme
        
        self.theme = Theme(self)
        return fallback

    def setContentLanguage(self, lang):
        """ Set the content language, used for the content div

        Actions that generate content in the user language, like search,
        should set the content direction to the user language before they
        call send_title!
        """
        self.content_lang = lang
        self.current_lang = lang

    def getPragma(self, key, defval=None):
        """ Query a pragma value (#pragma processing instruction)

            Keys are not case-sensitive.
        """
        return self.pragma.get(key.lower(), defval)

    def setPragma(self, key, value):
        """ Set a pragma value (#pragma processing instruction)

            Keys are not case-sensitive.
        """
        self.pragma[key.lower()] = value

    def getPathinfo(self):
        """ Return the remaining part of the URL. """
        return self.path_info

    def getScriptname(self):
        """ Return the scriptname part of the URL ('/path/to/my.cgi'). """
        if self.script_name == '/':
            return ''
        return self.script_name

    def getKnownActions(self):
        """ Create a dict of avaiable actions

        Return cached version if avaiable.
       
        @rtype: dict
        @return: dict of all known actions
        """
        try:
            self.cfg._known_actions # check
        except AttributeError:
            from MoinMoin import wikiaction
            # Add built in  actions from wikiaction
            actions = [name[3:] for name in wikiaction.__dict__ if name.startswith('do_')]

            # Add plugins           
            dummy, plugins = wikiaction.getPlugins(self)
            actions.extend(plugins)

            # Add extensions
            from MoinMoin.action import extension_actions
            actions.extend(extension_actions)           
           
            # TODO: Use set when we require Python 2.3
            actions = dict(zip(actions, [''] * len(actions)))            
            self.cfg._known_actions = actions

        # Return a copy, so clients will not change the dict.
        return self.cfg._known_actions.copy()        

    def getAvailableActions(self, page):
        """ Get list of avaiable actions for this request

        The dict does not contain actions that starts with lower case.
        Themes use this dict to display the actions to the user.

        @param page: current page, Page object
        @rtype: dict
        @return: dict of avaiable actions
        """
        if self._available_actions is None:
            # Add actions for existing pages only, including deleted pages.
            # Fix *OnNonExistingPage bugs.
            if not (page.exists(includeDeleted=1) and self.user.may.read(page.page_name)):
                return []

            # Filter non ui actions (starts with lower case letter)
            actions = self.getKnownActions()
            for key in actions.keys():
                if key[0].islower():
                    del actions[key]

            # Filter wiki excluded actions
            for key in self.cfg.actions_excluded:
                if key in actions:
                    del actions[key]                

            # Filter actions by page type, acl and user state
            excluded = []
            if ((page.isUnderlayPage() and not page.isStandardPage()) or
                not self.user.may.write(page.page_name) or
                not self.user.may.delete(page.page_name)):
                # Prevent modification of underlay only pages, or pages
                # the user can't write and can't delete
                excluded = [u'RenamePage', u'DeletePage', ] # AttachFile must NOT be here!
            for key in excluded:
                if key in actions:
                    del actions[key]                

            self._available_actions = actions

        # Return a copy, so clients will not change the dict.
        return self._available_actions.copy()

    def redirectedOutput(self, function, *args, **kw):
        """ Redirect output during function, return redirected output """
        buffer = StringIO.StringIO()
        self.redirect(buffer)
        try:
            function(*args, **kw)
        finally:
            self.redirect()
        text = buffer.getvalue()
        buffer.close()        
        return text

    def redirect(self, file=None):
        """ Redirect output to file, or restore saved output """
        if file:
            self.writestack.append(self.write)
            self.write = file.write
        else:
            self.write = self.writestack.pop()

    def reset_output(self):
        """ restore default output method
            destroy output stack
            (useful for error messages)
        """
        if self.writestack:
            self.write = self.writestack[0]
            self.writestack = []

    def log(self, msg):
        """ Log to stderr, which may be error.log """
        msg = msg.strip()
        # Encode unicode msg
        if isinstance(msg, unicode):
            msg = msg.encode(config.charset)
        # Add time stamp
        msg = '[%s] %s\n' % (time.asctime(), msg)
        sys.stderr.write(msg)

    def timing_log(self, start, action):
        """ Log to timing log (for performance analysis) """
        indicator = ''
        if start:
            total = "vvv"
        else:
            self.clock.stop('total') # make sure it is stopped
            total_secs = self.clock.timings['total']
            # we add some stuff that is easy to grep when searching for peformance problems:
            if total_secs > 50:
                indicator += '!4!'
            elif total_secs > 20:
               indicator += '!3!'
            elif total_secs > 10:
                indicator += '!2!'
            elif total_secs > 2:
                indicator += '!1!'
            total = self.clock.value('total')
            # use + for existing pages, - for non-existing pages
            indicator += self.page.exists() and '+' or '-'
            if self.isSpiderAgent:
                indicator += "B"

        # Add time stamp and process ID
        pid = os.getpid()
        t = time.time()
        timestr = time.strftime("%Y%m%d %H%M%S", time.gmtime(t))
        msg = '%s %5d %-6s %4s %-10s %s\n' % (timestr, pid, total, indicator, action, self.url)
        self.timing_logfile.write(msg)
        self.timing_logfile.flush()

    def write(self, *data):
        """ Write to output stream.
        """
        raise NotImplementedError

    def encode(self, data):
        """ encode data (can be both unicode strings and strings),
            preparing for a single write()
        """
        wd = []
        for d in data:
            try:
                if isinstance(d, unicode):
                    # if we are REALLY sure, we can use "strict"
                    d = d.encode(config.charset, 'replace')
                elif d is None:
                    continue
                wd.append(d)
            except UnicodeError:
                print >>sys.stderr, "Unicode error on: %s" % repr(d)
        return ''.join(wd)

    def decodePagename(self, name):
        """ Decode path, possibly using non ascii characters

        Does not change the name, only decode to Unicode.

        First split the path to pages, then decode each one. This enables
        us to decode one page using config.charset and another using
        utf-8. This situation happens when you try to add to a name of
        an existing page.

        See http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1
        
        @param name: page name, string
        @rtype: unicode
        @return decoded page name
        """
        # Split to pages and decode each one
        pages = name.split('/')
        decoded = []
        for page in pages:
            # Recode from utf-8 into config charset. If the path
            # contains user typed parts, they are encoded using 'utf-8'.
            if config.charset != 'utf-8':
                try:
                    page = unicode(page, 'utf-8', 'strict')
                    # Fit data into config.charset, replacing what won't
                    # fit. Better have few "?" in the name than crash.
                    page = page.encode(config.charset, 'replace')
                except UnicodeError:
                    pass
                
            # Decode from config.charset, replacing what can't be decoded.
            page = unicode(page, config.charset, 'replace')
            decoded.append(page)

        # Assemble decoded parts
        name = u'/'.join(decoded)
        return name

    def normalizePagename(self, name):
        """ Normalize page name 

        Convert '_' to spaces - allows using nice URLs with spaces, with no
        need to quote.

        Prevent creating page names with invisible characters or funny
        whitespace that might confuse the users or abuse the wiki, or
        just does not make sense.

        Restrict even more group pages, so they can be used inside acl lines.
        
        @param name: page name, unicode
        @rtype: unicode
        @return: decoded and sanitized page name
        """
        # Replace underscores with spaces
        name = name.replace(u'_', u' ')

        # Strip invalid characters
        name = config.page_invalid_chars_regex.sub(u'', name)

        # Split to pages and normalize each one
        pages = name.split(u'/')
        normalized = []
        for page in pages:            
            # Ignore empty or whitespace only pages
            if not page or page.isspace():
                continue

            # Cleanup group pages.
            # Strip non alpha numeric characters, keep white space
            if wikiutil.isGroupPage(self, page):
                page = u''.join([c for c in page
                                 if c.isalnum() or c.isspace()])

            # Normalize white space. Each name can contain multiple 
            # words separated with only one space. Split handle all
            # 30 unicode spaces (isspace() == True)
            page = u' '.join(page.split())
            
            normalized.append(page)            
        
        # Assemble components into full pagename
        name = u'/'.join(normalized)
        return name
        
    def read(self, n):
        """ Read n bytes from input stream.
        """
        raise NotImplementedError

    def flush(self):
        """ Flush output stream.
        """
        raise NotImplementedError

    def check_spider(self):
        """ check if the user agent for current request is a spider/bot """
        isSpider = False
        spiders = self.cfg.ua_spiders
        if spiders:
            ua = self.getUserAgent()
            if ua:
                isSpider = re.search(spiders, ua, re.I) is not None
        return isSpider

    def isForbidden(self):
        """ check for web spiders and refuse anything except viewing """
        forbidden = 0
        # we do not have a parsed query string here, so we can just do simple matching
        qs = self.query_string
        if ((qs != '' or self.request_method != 'GET') and
            not 'action=rss_rc' in qs and
            # allow spiders to get attachments and do 'show'
            not ('action=AttachFile' in qs and 'do=get' in qs) and
            not 'action=show' in qs and
            not 'action=sitemap' in qs
            ):
            forbidden = self.isSpiderAgent

        if not forbidden and self.cfg.hosts_deny:
            ip = self.remote_addr
            for host in self.cfg.hosts_deny:
                if host[-1] == '.' and ip.startswith(host):
                    forbidden = 1
                    #self.log("hosts_deny (net): %s" % str(forbidden))
                    break
                if ip == host:
                    forbidden = 1
                    #self.log("hosts_deny (ip): %s" % str(forbidden))
                    break
        return forbidden

    def setup_args(self, form=None):
        """ Return args dict 
        
        In POST request, invoke _setup_args_from_cgi_form to handle possible
        file uploads. For other request simply parse the query string.
        
        Warning: calling with a form might fail, depending on the type of the
        request! Only the request know which kind of form it can handle.
        
        TODO: The form argument should be removed in 1.5.
        """
        if form is not None or self.request_method == 'POST':
            return self._setup_args_from_cgi_form(form)
        args = cgi.parse_qs(self.query_string, keep_blank_values=1)
        return self.decodeArgs(args)

    def _setup_args_from_cgi_form(self, form=None):
        """ Return args dict from a FieldStorage
        
        Create the args from a standard cgi.FieldStorage or from given form.
        Each key contain a list of values.

        @param form: a cgi.FieldStorage
        @rtype: dict
        @return: dict with form keys, each contains a list of values
        """
        if form is None:
            form = cgi.FieldStorage()

        args = {}
        for key in form:
            values = form[key]
            if not isinstance(values, list):
                values = [values]
            fixedResult = []
            for item in values:
                fixedResult.append(item.value)
                if isinstance(item, cgi.FieldStorage) and item.filename:
                    # Save upload file name in a separate key
                    args[key + '__filename__'] = item.filename            
            args[key] = fixedResult
            
        return self.decodeArgs(args)

    def decodeArgs(self, args):
        """ Decode args dict 
        
        Decoding is done in a separate path because it is reused by
        other methods and sub classes.
        """
        decode = wikiutil.decodeUserInput
        result = {}
        for key in args:
            if key + '__filename__' in args:
                # Copy file data as is
                result[key] = args[key]
            elif key.endswith('__filename__'):
                result[key] = decode(args[key], self.decode_charsets)
            else:
                result[key] = [decode(value, self.decode_charsets) for value in args[key]]
        return result

    def getBaseURL(self):
        """ Return a fully qualified URL to this script. """
        return self.getQualifiedURL(self.getScriptname())

    def getQualifiedURL(self, uri=''):
        """ Return an absolute URL starting with schema and host.

        Already qualified urls are returned unchanged.

        @param uri: server rooted uri e.g /scriptname/pagename.
                    It must start with a slash. Must be ascii and url encoded.
        """
        import urlparse
        scheme = urlparse.urlparse(uri)[0]
        if scheme:
            return uri

        scheme = ('http', 'https')[self.is_ssl]
        result = "%s://%s%s" % (scheme, self.http_host, uri)

        # This might break qualified urls in redirects!
        # e.g. mapping 'http://netloc' -> '/'
        return wikiutil.mapURL(self, result)

    def getUserAgent(self):
        """ Get the user agent. """
        return self.http_user_agent

    def makeForbidden(self, resultcode, msg):
        statusmsg = {
            403: 'FORBIDDEN',
            404: 'Not found',
            503: 'Service unavailable',
        }
        self.http_headers([
            'Status: %d %s' % (resultcode, statusmsg[resultcode]),
            'Content-Type: text/plain'
        ])
        self.write(msg)
        self.setResponseCode(resultcode)
        self.forbidden = True

    def makeForbidden403(self):
        self.makeForbidden(403, 'You are not allowed to access this!\r\n')

    def makeUnavailable503(self):
        self.makeForbidden(503, "Warning:\r\n"
                   "You triggered the wiki's surge protection by doing too many requests in a short time.\r\n"
                   "Please make a short break reading the stuff you already got.\r\n"
                   "When you restart doing requests AFTER that, slow down or you might get locked out for a longer time!\r\n")

    def initTheme(self):
        """ Set theme - forced theme, user theme or wiki default """
        if self.cfg.theme_force:
            theme_name = self.cfg.theme_default
        else:
            theme_name = self.user.theme_name
        self.loadTheme(theme_name)
        
    def run(self):
        # Exit now if __init__ failed or request is forbidden
        if self.failed or self.forbidden:
            # Don't sleep() here, it binds too much of our resources!
            return self.finish()

        _ = self.getText
        self.clock.start('run')

        from MoinMoin.Page import Page

        if self.query_string == 'action=xmlrpc':
            from MoinMoin.wikirpc import xmlrpc
            xmlrpc(self)
            return self.finish()
        
        if self.query_string == 'action=xmlrpc2':
            from MoinMoin.wikirpc import xmlrpc2
            xmlrpc2(self)
            return self.finish()

        # parse request data
        try:
            self.initTheme()
            
            action = self.form.get('action', [None])[0]

            if self.cfg.log_timing:
                self.timing_log(True, action)

            # The last component in path_info is the page name, if any
            path = self.getPathinfo()
            if path.startswith('/'):
                pagename = self.normalizePagename(path)
            else:
                pagename = None

            # need to inform caches that content changes based on:
            # * cookie (even if we aren't sending one now)
            # * User-Agent (because a bot might be denied and get no content)
            # * Accept-Language (except if moin is told to ignore browser language)
            if self.cfg.language_ignore_browser:
                self.setHttpHeader("Vary: Cookie,User-Agent")
            else:
                self.setHttpHeader("Vary: Cookie,User-Agent,Accept-Language")

            # Handle request. We have these options:
            
            # 1. If user has a bad user name, delete its bad cookie and
            # send him to UserPreferences to make a new account.
            if not user.isValidName(self, self.user.name):
                msg = _("""Invalid user name {{{'%s'}}}.
Name may contain any Unicode alpha numeric character, with optional one
space between words. Group page name is not allowed.""") % self.user.name
                self.user = self.get_user_default_unknown(name=self.user.name, logout=True)
                page = wikiutil.getSysPage(self, 'UserPreferences')
                page.send_page(self, msg=msg)

            # 2. Or jump to page where user left off
            elif not pagename and not action and self.user.remember_last_visit:
                pagetrail = self.user.getTrail()
                if pagetrail:
                    # Redirect to last page visited
                    if ":" in pagetrail[-1]:
                        wikitag, wikiurl, wikitail, error = wikiutil.resolve_wiki(self, pagetrail[-1]) 
                        url = wikiurl + wikiutil.quoteWikinameURL(wikitail)
                    else:
                        url = Page(self, pagetrail[-1]).url(self)
                else:
                    # Or to localized FrontPage
                    url = wikiutil.getFrontPage(self).url(self)
                self.http_redirect(url)
                return self.finish()
            
            # 3. Or save drawing
            elif self.form.has_key('filepath') and self.form.has_key('noredirect'):
                # looks like user wants to save a drawing
                from MoinMoin.action.AttachFile import execute
                # TODO: what if pagename is None?
                execute(pagename, self)
                raise MoinMoinNoFooter           

            # 4. Or handle action
            else:
                if action is None:
                    action = 'show'
                # pagename could be empty after normalization e.g. '///' -> ''
                # Use localized FrontPage if pagename is empty
                if not pagename:
                    self.page = wikiutil.getFrontPage(self)
                else:
                    self.page = Page(self, pagename)

                # Complain about unknown actions
                if not action in self.getKnownActions():
                    self.http_headers()
                    self.write(u'<html><body><h1>Unknown action %s</h1></body>' % wikiutil.escape(action))

                # Disallow non available actions
                elif action[0].isupper() and not action in self.getAvailableActions(self.page):
                    # Send page with error
                    msg = _("You are not allowed to do %s on this page.") % wikiutil.escape(action)
                    if not self.user.valid:
                        # Suggest non valid user to login
                        msg += " " + _("Login and try again.", formatted=0)
                    self.page.send_page(self, msg=msg)

                # Try action
                else:
                    from MoinMoin.wikiaction import getHandler
                    handler = getHandler(self, action)
                    if handler is None:
                        # Send page with error
                        msg = _("You are not allowed to do %s on this page.") % wikiutil.escape(action)
                        if not self.user.valid:
                            # Suggest non valid user to login
                            msg += " " + _("Login and try again.", formatted=0)
                        self.page.send_page(self, msg=msg)
                    else:
                        handler(self.page.page_name, self)

            # generate page footer (actions that do not want this footer use
            # raise util.MoinMoinNoFooter to break out of the default execution
            # path, see the "except MoinMoinNoFooter" below)

            self.clock.stop('run')
            self.clock.stop('total')

            # Close html code
            if self.cfg.show_timings and action != 'print':
                self.write('<ul id="timings">\n')
                for t in self.clock.dump():
                    self.write('<li>%s</li>\n' % t)
                self.write('</ul>\n')
            #self.write('<!-- auth_method == %s -->' % repr(self.user.auth_method))
            self.write('</body>\n</html>\n\n')
            
        except MoinMoinNoFooter:
            pass
        except Exception, err:
            self.fail(err)

        if self.cfg.log_timing:
            self.timing_log(False, action)

        return self.finish()

    def http_redirect(self, url):
        """ Redirect to a fully qualified, or server-rooted URL
        
        @param url: relative or absolute url, ascii using url encoding.
        """
        url = self.getQualifiedURL(url)
        self.http_headers(["Status: 302 Found", "Location: %s" % url])
        self.setResponseCode(302)

    def setHttpHeader(self, header):
        """ Save header for later send.
        
            Attention: although we use a list here, some implementations use a dict,
            thus multiple calls with the same header type do NOT work in the end!
        """
        self.user_headers.append(header)

    def setResponseCode(self, code, message=None):
        pass

    def fail(self, err):
        """ Fail when we can't continue

        Send 500 status code with the error name. Reference: 
        http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1

        Log the error, then let failure module handle it. 

        @param err: Exception instance or subclass.
        """
        self.failed = 1 # save state for self.run()            
        self.http_headers(['Status: 500 MoinMoin Internal Error'])
        self.setResponseCode(500)
        self.log('%s: %s' % (err.__class__.__name__, str(err)))        
        from MoinMoin import failure
        failure.handle(self)             

    def open_logs(self):
        pass

    def makeUniqueID(self, base):
        """
        Generates a unique ID using a given base name. Appends a running count to the base.

        @param base: the base of the id
        @type base: unicode

        @returns: an unique id
        @rtype: unicode
        """
        if not isinstance(base, unicode):
            base = unicode(str(base), 'ascii', 'ignore')
        count = self._page_ids.get(base, -1) + 1
        self._page_ids[base] = count
        if count == 0:
            return base
        return u'%s_%04d' % (base, count)

    def httpDate(self, when=None, rfc='1123'):
        """ Returns http date string, according to rfc2068

        See http://www.cse.ohio-state.edu/cgi-bin/rfc/rfc2068.html#sec-3.3

        A http 1.1 server should use only rfc1123 date, but cookie's
        "expires" field should use the older obsolete rfc850 date.

        Note: we can not use strftime() because that honors the locale
        and rfc2822 requires english day and month names.

        We can not use email.Utils.formatdate because it formats the
        zone as '-0000' instead of 'GMT', and creates only rfc1123
        dates. This is a modified version of email.Utils.formatdate
        from Python 2.4.

        @param when: seconds from epoch, as returned by time.time()
        @param rfc: conform to rfc ('1123' or '850')
        @rtype: string
        @return: http date conforming to rfc1123 or rfc850
        """
        if when is None:
            when = time.time()
        now = time.gmtime(when)
        month = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul',
                 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][now.tm_mon - 1]
        if rfc == '1123':
            day = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][now.tm_wday]
            date = '%02d %s %04d' % (now.tm_mday, month, now.tm_year)
        elif rfc == '850':
            day = ["Monday", "Tuesday", "Wednesday", "Thursday",
                    "Friday", "Saturday", "Sunday"][now.tm_wday]
            date = '%02d-%s-%s' % (now.tm_mday, month, str(now.tm_year)[-2:])
        else:
            raise ValueError("Invalid rfc value: %s" % rfc)
        
        return '%s, %s %02d:%02d:%02d GMT' % (day, date, now.tm_hour,
                                              now.tm_min, now.tm_sec)

    def disableHttpCaching(self, level=1):
        """ Prevent caching of pages that should not be cached.

        level == 1 means disabling caching when we have a cookie set
        level == 2 means completely disabling caching (used by Page*Editor)

        This is important to prevent caches break acl by providing one
        user pages meant to be seen only by another user, when both users
        share the same caching proxy.

        AVOID using no-cache and no-store for attachments as it is completely broken on IE!
        
        Details: http://support.microsoft.com/support/kb/articles/Q234/0/67.ASP
        """
        if level <= self.http_caching_disabled:
            return # only make caching stricter

        if level == 1:
            # Set Cache control header for http 1.1 caches
            # See http://www.cse.ohio-state.edu/cgi-bin/rfc/rfc2109.html#sec-4.2.3
            # and http://www.cse.ohio-state.edu/cgi-bin/rfc/rfc2068.html#sec-14.9
            #self.setHttpHeader('Cache-Control: no-cache="set-cookie", private, max-age=0')
            self.setHttpHeader('Cache-Control: private, must-revalidate, max-age=10')
        elif level == 2:
            self.setHttpHeader('Cache-Control: no-cache')

        # Set Expires for http 1.0 caches (does not support Cache-Control)
        when = time.time() - (3600 * 24 * 365)
        self.setHttpHeader('Expires: %s' % self.httpDate(when=when))

        # Set Pragma for http 1.0 caches
        # See http://www.cse.ohio-state.edu/cgi-bin/rfc/rfc2068.html#sec-14.32
        # DISABLED for level == 1 to fix IE https file attachment downloading trouble.
        if level == 2:
            self.setHttpHeader('Pragma: no-cache')

        self.http_caching_disabled = level

    def finish(self):
        """ General cleanup on end of request
        
        Delete circular references - all object that we create using self.name = class(self).
        This helps Python to collect these objects and keep our memory footprint lower.
        """
        try:
            del self.user
            del self.theme
            del self.dicts
        except:
            pass

    # Debug ------------------------------------------------------------

    def debugEnvironment(self, env):
        """ Environment debugging aid """
        # Keep this one name per line so its easy to comment stuff
        names = [
#             'http_accept_language',
#             'http_host',
#             'http_referer',
#             'http_user_agent',
#             'is_ssl',
            'path_info',
            'query_string',
#             'remote_addr',
            'request_method',
#             'request_uri',
#             'saved_cookie',
            'script_name',
#             'server_name',
#             'server_port',
            ]
        names.sort()
        attributes = []
        for name in names:
            attributes.append('  %s = %r\n' % (name, getattr(self, name, None)))
        attributes = ''.join(attributes)
        
        environment = []
        names = env.keys()
        names.sort()
        for key in names:
            environment.append('  %s = %r\n' % (key, env[key]))
        environment = ''.join(environment)
        
        data = '\nRequest Attributes\n%s\nEnviroment\n%s' % (attributes, environment)        
        f = open('/tmp/env.log', 'a')
        try:
            f.write(data)
        finally:
            f.close()
  

# CGI ---------------------------------------------------------------

class RequestCGI(RequestBase):
    """ specialized on CGI requests """

    def __init__(self, properties={}):
        try:
            # force input/output to binary
            if sys.platform == "win32":
                import msvcrt
                msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
                msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)

            self._setup_vars_from_std_env(os.environ)
            RequestBase.__init__(self, properties)

        except Exception, err:
            self.fail(err)
            
    def open_logs(self):
        # create log file for catching stderr output
        if not self.opened_logs:
            sys.stderr = open(os.path.join(self.cfg.data_dir, 'error.log'), 'at', 0)
            self.opened_logs = 1

    def read(self, n=None):
        """ Read from input stream. """
        if n is None:
            return sys.stdin.read()
        else:
            return sys.stdin.read(n)

    def write(self, *data):
        """ Write to output stream. """
        sys.stdout.write(self.encode(data))

    def flush(self):
        sys.stdout.flush()
        
    def finish(self):
        RequestBase.finish(self)
        # flush the output, ignore errors caused by the user closing the socket
        try:
            sys.stdout.flush()
        except IOError, ex:
            import errno
            if ex.errno != errno.EPIPE: raise

    # Headers ----------------------------------------------------------
    
    def http_headers(self, more_headers=[]):
        # Send only once
        if getattr(self, 'sent_headers', None):
            return
        
        self.sent_headers = 1
        have_ct = 0

        # send http headers
        for header in more_headers + getattr(self, 'user_headers', []):
            if header.lower().startswith("content-type:"):
                # don't send content-type multiple times!
                if have_ct: continue
                have_ct = 1
            if type(header) is unicode:
                header = header.encode('ascii')
            self.write("%s\r\n" % header)

        if not have_ct:
            self.write("Content-type: text/html;charset=%s\r\n" % config.charset)

        self.write('\r\n')

        #from pprint import pformat
        #sys.stderr.write(pformat(more_headers))
        #sys.stderr.write(pformat(self.user_headers))


# Twisted -----------------------------------------------------------

class RequestTwisted(RequestBase):
    """ specialized on Twisted requests """

    def __init__(self, twistedRequest, pagename, reactor, properties={}):
        try:
            self.twistd = twistedRequest
            self.reactor = reactor
            
            # Copy headers
            self.http_accept_language = self.twistd.getHeader('Accept-Language')
            self.saved_cookie = self.twistd.getHeader('Cookie')
            self.http_user_agent = self.twistd.getHeader('User-Agent')
            self.if_modified_since = self.twistd.getHeader('If-Modified-Since')
            self.if_none_match = self.twistd.getHeader('If-None-Match')
           
            # Copy values from twisted request
            self.server_protocol = self.twistd.clientproto
            self.server_name = self.twistd.getRequestHostname().split(':')[0]
            self.server_port = str(self.twistd.getHost()[2])
            self.is_ssl = self.twistd.isSecure()
            self.path_info = '/' + '/'.join([pagename] + self.twistd.postpath)
            self.request_method = self.twistd.method
            self.remote_addr = self.twistd.getClientIP()
            self.request_uri = self.twistd.uri
            self.script_name = "/" + '/'.join(self.twistd.prepath[:-1])

            # Values that need more work
            self.query_string = self.splitURI(self.twistd.uri)[1]
            self.setHttpReferer(self.twistd.getHeader('Referer'))
            self.setHost()
            self.setURL(self.twistd.getAllHeaders())

            ##self.debugEnvironment(twistedRequest.getAllHeaders())
            
            RequestBase.__init__(self, properties)

        except MoinMoinNoFooter: # might be triggered by http_redirect
            self.http_headers() # send headers (important for sending MOIN_ID cookie)
            self.finish()

        except Exception, err:
            # Wrap err inside an internal error if needed
            from MoinMoin import error
            if isinstance(err, error.FatalError):
                self.delayedError = err
            else:
                self.delayedError = error.InternalError(str(err))

    def run(self):
        """ Handle delayed errors then invoke base class run """
        if hasattr(self, 'delayedError'):
            self.fail(self.delayedError)
            return self.finish()
        RequestBase.run(self)
            
    def setup_args(self, form=None):
        """ Return args dict 
        
        Twisted already parsed args, including __filename__ hacking,
        but did not decoded the values.
        """
        return self.decodeArgs(self.twistd.args)
        
    def read(self, n=None):
        """ Read from input stream. """
        # XXX why is that wrong?:
        #rd = self.reactor.callFromThread(self.twistd.read)
        
        # XXX do we need self.reactor.callFromThread with that?
        # XXX if yes, why doesn't it work?
        self.twistd.content.seek(0, 0)
        if n is None:
            rd = self.twistd.content.read()
        else:
            rd = self.twistd.content.read(n)
        #print "request.RequestTwisted.read: data=\n" + str(rd)
        return rd
    
    def write(self, *data):
        """ Write to output stream. """
        #print "request.RequestTwisted.write: data=\n" + wd
        self.reactor.callFromThread(self.twistd.write, self.encode(data))

    def flush(self):
        pass # XXX is there a flush in twisted?

    def finish(self):
        RequestBase.finish(self)
        self.reactor.callFromThread(self.twistd.finish)

    def open_logs(self):
        return
        # create log file for catching stderr output
        if not self.opened_logs:
            sys.stderr = open(os.path.join(self.cfg.data_dir, 'error.log'), 'at', 0)
            self.opened_logs = 1

    # Headers ----------------------------------------------------------

    def __setHttpHeader(self, header):
        if type(header) is unicode:
            header = header.encode('ascii')
        key, value = header.split(':', 1)
        value = value.lstrip()
        if key.lower() == 'set-cookie':
            key, value = value.split('=', 1)
            self.twistd.addCookie(key, value)
        else:
            self.twistd.setHeader(key, value)
        #print "request.RequestTwisted.setHttpHeader: %s" % header

    def http_headers(self, more_headers=[]):
        if getattr(self, 'sent_headers', None):
            return
        self.sent_headers = 1
        have_ct = 0

        # set http headers
        for header in more_headers + getattr(self, 'user_headers', []):
            if header.lower().startswith("content-type:"):
                # don't send content-type multiple times!
                if have_ct: continue
                have_ct = 1
            self.__setHttpHeader(header)

        if not have_ct:
            self.__setHttpHeader("Content-type: text/html;charset=%s" % config.charset)

    def http_redirect(self, url):
        """ Redirect to a fully qualified, or server-rooted URL 
        
        @param url: relative or absolute url, ascii using url encoding.
        """
        url = self.getQualifiedURL(url)
        self.twistd.redirect(url)
        # calling finish here will send the rest of the data to the next
        # request. leave the finish call to run()
        #self.twistd.finish()
        raise MoinMoinNoFooter

    def setResponseCode(self, code, message=None):
        self.twistd.setResponseCode(code, message)
        
# CLI ------------------------------------------------------------------

class RequestCLI(RequestBase):
    """ specialized on command line interface and script requests """

    def __init__(self, url='CLI', pagename='', properties={}):
        self.saved_cookie = ''
        self.path_info = '/' + pagename
        self.query_string = ''
        self.remote_addr = '127.0.0.1'
        self.is_ssl = 0
        self.http_user_agent = 'CLI/Script'
        self.url = url
        self.request_method = 'GET'
        self.request_uri = '/' + pagename # TODO check
        self.http_host = 'localhost'
        self.http_referer = ''
        self.script_name = '.'
        self.if_modified_since = None
        self.if_none_match = None
        RequestBase.__init__(self, properties)
        self.cfg.caching_formats = [] # don't spoil the cache
        self.initTheme() # usually request.run() does this, but we don't use it
  
    def read(self, n=None):
        """ Read from input stream. """
        if n is None:
            return sys.stdin.read()
        else:
            return sys.stdin.read(n)

    def write(self, *data):
        """ Write to output stream. """
        sys.stdout.write(self.encode(data))

    def flush(self):
        sys.stdout.flush()
        
    def finish(self):
        RequestBase.finish(self)
        # flush the output, ignore errors caused by the user closing the socket
        try:
            sys.stdout.flush()
        except IOError, ex:
            import errno
            if ex.errno != errno.EPIPE: raise

    def isForbidden(self):
        """ Nothing is forbidden """
        return 0

    # Accessors --------------------------------------------------------

    def getQualifiedURL(self, uri=None):
        """ Return a full URL starting with schema and host
        
        TODO: does this create correct pages when you render wiki pages
              within a cli request?!
        """
        return uri

    # Headers ----------------------------------------------------------

    def setHttpHeader(self, header):
        pass

    def http_headers(self, more_headers=[]):
        pass

    def http_redirect(self, url):
        """ Redirect to a fully qualified, or server-rooted URL 
        
        TODO: Does this work for rendering redirect pages?
        """
        raise Exception("Redirect not supported for command line tools!")


# StandAlone Server ----------------------------------------------------

class RequestStandAlone(RequestBase):
    """ specialized on StandAlone Server (MoinMoin.server.standalone) requests """
    script_name = ''
    
    def __init__(self, sa, properties={}):
        """
        @param sa: stand alone server object
        @param properties: ...
        """
        try:
            self.sareq = sa
            self.wfile = sa.wfile
            self.rfile = sa.rfile
            self.headers = sa.headers
            self.is_ssl = 0
            
            # Copy headers
            self.http_accept_language = (sa.headers.getheader('accept-language') 
                                         or self.http_accept_language)
            self.http_user_agent = sa.headers.getheader('user-agent', '')            
            co = filter(None, sa.headers.getheaders('cookie'))
            self.saved_cookie = ', '.join(co) or ''
            self.if_modified_since = sa.headers.getheader('if-modified-since')
            self.if_none_match = sa.headers.getheader('if-none-match')
            
            # Copy rest from standalone request   
            self.server_name = sa.server.server_name
            self.server_port = str(sa.server.server_port)
            self.request_method = sa.command
            self.request_uri = sa.path
            self.remote_addr = sa.client_address[0]

            # Values that need more work                        
            self.path_info, self.query_string = self.splitURI(sa.path)
            self.setHttpReferer(sa.headers.getheader('referer'))
            self.setHost(sa.headers.getheader('host'))
            self.setURL(sa.headers)

            ##self.debugEnvironment(sa.headers)
            
            RequestBase.__init__(self, properties)

        except Exception, err:
            self.fail(err)

    def _setup_args_from_cgi_form(self, form=None):
        """ Override to create standlone form """
        form = cgi.FieldStorage(self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'})
        return RequestBase._setup_args_from_cgi_form(self, form)
        
    def read(self, n=None):
        """ Read from input stream
        
        Since self.rfile.read() will block, content-length will be used instead.
        
        TODO: test with n > content length, or when calling several times
        with smaller n but total over content length.
        """
        if n is None:
            try:
                n = int(self.headers.get('content-length'))
            except (TypeError, ValueError):
                import warnings
                warnings.warn("calling request.read() when content-length is "
                              "not available will block")
                return self.rfile.read()
        return self.rfile.read(n)

    def write(self, *data):
        """ Write to output stream. """
        self.wfile.write(self.encode(data))

    def flush(self):
        self.wfile.flush()
        
    def finish(self):
        RequestBase.finish(self)
        self.wfile.flush()

    # Headers ----------------------------------------------------------

    def http_headers(self, more_headers=[]):
        if getattr(self, 'sent_headers', None):
            return
        
        self.sent_headers = 1
        user_headers = getattr(self, 'user_headers', [])
        
        # check for status header and send it
        our_status = 200
        for header in more_headers + user_headers:
            if header.lower().startswith("status:"):
                try:
                    our_status = int(header.split(':', 1)[1].strip().split(" ", 1)[0]) 
                except:
                    pass
                # there should be only one!
                break
        # send response
        self.sareq.send_response(our_status)

        # send http headers
        have_ct = 0
        for header in more_headers + user_headers:
            if type(header) is unicode:
                header = header.encode('ascii')
            if header.lower().startswith("content-type:"):
                # don't send content-type multiple times!
                if have_ct: continue
                have_ct = 1

            self.write("%s\r\n" % header)

        if not have_ct:
            self.write("Content-type: text/html;charset=%s\r\n" % config.charset)

        self.write('\r\n')

        #from pprint import pformat
        #sys.stderr.write(pformat(more_headers))
        #sys.stderr.write(pformat(self.user_headers))

# mod_python/Apache ----------------------------------------------------

class RequestModPy(RequestBase):
    """ specialized on mod_python requests """

    def __init__(self, req):
        """ Saves mod_pythons request and sets basic variables using
            the req.subprocess_env, cause this provides a standard
            way to access the values we need here.

            @param req: the mod_python request instance
        """
        try:
            # flags if headers sent out contained content-type or status
            self._have_ct = 0
            self._have_status = 0

            req.add_common_vars()
            self.mpyreq = req
            # some mod_python 2.7.X has no get method for table objects,
            # so we make a real dict out of it first.
            if not hasattr(req.subprocess_env, 'get'):
                env=dict(req.subprocess_env)
            else:
                env=req.subprocess_env
            self._setup_vars_from_std_env(env)
            RequestBase.__init__(self)

        except Exception, err:
            self.fail(err)
            
    def fixURI(self, env):
        """ Fix problems with script_name and path_info using
        PythonOption directive to rewrite URI.
        
        This is needed when using Apache 1 or other server which does
        not support adding custom headers per request. With mod_python we
        can use the PythonOption directive:
        
            <Location /url/to/mywiki/>
                PythonOption X-Moin-Location /url/to/mywiki/
            </location>

        Note that *neither* script_name *nor* path_info can be trusted
        when Moin is invoked as a mod_python handler with apache1, so we
        must build both using request_uri and the provided PythonOption.
        """
        # Be compatible with release 1.3.5 "Location" option 
        # TODO: Remove in later release, we should have one option only.
        old_location = 'Location'
        options_table = self.mpyreq.get_options()
        if not hasattr(options_table, 'get'):
            options = dict(options_table)
        else:
            options = options_table
        location = options.get(self.moin_location) or options.get(old_location)
        if location:
            env[self.moin_location] = location
            # Try to recreate script_name and path_info from request_uri.
            import urlparse
            scriptAndPath = urlparse.urlparse(self.request_uri)[2]
            self.script_name = location.rstrip('/')
            path = scriptAndPath.replace(self.script_name, '', 1)            
            self.path_info = wikiutil.url_unquote(path, want_unicode=False)

        RequestBase.fixURI(self, env)

    def _setup_args_from_cgi_form(self, form=None):
        """ Override to use mod_python.util.FieldStorage 
        
        Its little different from cgi.FieldStorage, so we need to
        duplicate the conversion code.
        """
        from mod_python import util
        if form is None:
            form = util.FieldStorage(self.mpyreq)

        args = {}
        for key in form.keys():
            if key is None:
                continue                
            values = form[key]
            if not isinstance(values, list):
                values = [values]
            fixedResult = []

            for item in values:
                # Remember filenames with a name hack
                if hasattr(item, 'filename') and item.filename:
                    args[key + '__filename__'] = item.filename
                # mod_python 2.7 might return strings instead of Field
                # objects.
                if hasattr(item, 'value'):
                    item = item.value
                fixedResult.append(item)                
            args[key] = fixedResult
            
        return self.decodeArgs(args)

    def run(self, req):
        """ mod_python calls this with its request object. We don't
            need it cause its already passed to __init__. So ignore
            it and just return RequestBase.run.

            @param req: the mod_python request instance
        """
        return RequestBase.run(self)

    def read(self, n=None):
        """ Read from input stream. """
        if n is None:
            return self.mpyreq.read()
        else:
            return self.mpyreq.read(n)

    def write(self, *data):
        """ Write to output stream. """
        self.mpyreq.write(self.encode(data))

    def flush(self):
        """ We can't flush it, so do nothing. """
        pass
        
    def finish(self):
        """ Just return apache.OK. Status is set in req.status. """
        RequestBase.finish(self)
        # is it possible that we need to return something else here?
        from mod_python import apache
        return apache.OK

    # Headers ----------------------------------------------------------

    def setHttpHeader(self, header):
        """ Filters out content-type and status to set them directly
            in the mod_python request. Rest is put into the headers_out
            member of the mod_python request.

            @param header: string, containing valid HTTP header.
        """
        if type(header) is unicode:
            header = header.encode('ascii')
        key, value = header.split(':', 1)
        value = value.lstrip()
        if key.lower() == 'content-type':
            # save content-type for http_headers
            if not self._have_ct:
                # we only use the first content-type!
                self.mpyreq.content_type = value
                self._have_ct = 1
        elif key.lower() == 'status':
            # save status for finish
            try:
                self.mpyreq.status = int(value.split(' ', 1)[0])
            except:
                pass
            else:
                self._have_status = 1
        else:
            # this is a header we sent out
            self.mpyreq.headers_out[key]=value

    def http_headers(self, more_headers=[]):
        """ Sends out headers and possibly sets default content-type
            and status.

            @param more_headers: list of strings, defaults to []
        """
        for header in more_headers + getattr(self, 'user_headers', []):
            self.setHttpHeader(header)
        # if we don't had an content-type header, set text/html
        if self._have_ct == 0:
            self.mpyreq.content_type = "text/html;charset=%s" % config.charset
        # if we don't had a status header, set 200
        if self._have_status == 0:
            self.mpyreq.status = 200
        # this is for mod_python 2.7.X, for 3.X it's a NOP
        self.mpyreq.send_http_header()

# FastCGI -----------------------------------------------------------

class RequestFastCGI(RequestBase):
    """ specialized on FastCGI requests """

    def __init__(self, fcgRequest, env, form, properties={}):
        """ Initializes variables from FastCGI environment and saves
            FastCGI request and form for further use.

            @param fcgRequest: the FastCGI request instance.
            @param env: environment passed by FastCGI.
            @param form: FieldStorage passed by FastCGI.
        """
        try:
            self.fcgreq = fcgRequest
            self.fcgenv = env
            self.fcgform = form
            self._setup_vars_from_std_env(env)
            RequestBase.__init__(self, properties)

        except Exception, err:
            self.fail(err)

    def _setup_args_from_cgi_form(self, form=None):
        """ Override to use FastCGI form """
        if form is None:
            form = self.fcgform
        return RequestBase._setup_args_from_cgi_form(self, form)

    def open_logs(self):
        # create log file for catching stderr output
        if not self.opened_logs:
            sys.stderr = open(os.path.join(self.cfg.data_dir, 'error.log'), 'at', 0)
            self.opened_logs = 1

    def read(self, n=None):
        """ Read from input stream. """
        if n is None:
            return self.fcgreq.stdin.read()
        else:
            return self.fcgreq.stdin.read(n)

    def write(self, *data):
        """ Write to output stream. """
        self.fcgreq.out.write(self.encode(data))

    def flush(self):
        """ Flush output stream. """
        self.fcgreq.flush_out()

    def finish(self):
        """ Call finish method of FastCGI request to finish handling of this request. """
        RequestBase.finish(self)
        self.fcgreq.finish()

    # Headers ----------------------------------------------------------

    def http_headers(self, more_headers=[]):
        """ Send out HTTP headers. Possibly set a default content-type. """
        if getattr(self, 'sent_headers', None):
            return
        self.sent_headers = 1
        have_ct = 0

        # send http headers
        for header in more_headers + getattr(self, 'user_headers', []):
            if type(header) is unicode:
                header = header.encode('ascii')
            if header.lower().startswith("content-type:"):
                # don't send content-type multiple times!
                if have_ct: continue
                have_ct = 1
            self.write("%s\r\n" % header)

        if not have_ct:
            self.write("Content-type: text/html;charset=%s\r\n" % config.charset)

        self.write('\r\n')

        #from pprint import pformat
        #sys.stderr.write(pformat(more_headers))
        #sys.stderr.write(pformat(self.user_headers))

    def open_logs(self):
        # create log file for catching stderr output
        if not self.opened_logs:
            sys.stderr = open(os.path.join(self.cfg.data_dir, 'error.log'), 'at')
            if self.cfg.log_timing:
                self.timing_logfile = open(os.path.join(self.cfg.data_dir, 'timing.log'), 'at')
            self.opened_logs = 1

# WSGI --------------------------------------------------------------

class RequestWSGI(RequestBase):
    def __init__(self, env):
        try:
            self.env = env
            self.hasContentType = False
            
            self.stdin = env['wsgi.input']
            self.stdout = StringIO.StringIO()
            
            self.status = '200 OK'
            self.headers = []
            
            self._setup_vars_from_std_env(env)
            RequestBase.__init__(self, {})
        
        except Exception, err:
            self.fail(err)
    
    def setup_args(self, form=None):
        if form is None:
            form = cgi.FieldStorage(fp=self.stdin, environ=self.env, keep_blank_values=1)
        return self._setup_args_from_cgi_form(form)
    
    def read(self, n=None):
        if n is None:
            return self.stdin.read()
        else:
            return self.stdin.read(n)
    
    def write(self, *data):
        self.stdout.write(self.encode(data))
    
    def reset_output(self):
        self.stdout = StringIO.StringIO()
    
    def setHttpHeader(self, header):
        if type(header) is unicode:
            header = header.encode('ascii')
        
        key, value = header.split(':', 1)
        value = value.lstrip()
        if key.lower() == 'content-type':
            # save content-type for http_headers
            if self.hasContentType:
                # we only use the first content-type!
                return
            else:
                self.hasContentType = True
        
        elif key.lower() == 'status':
            # save status for finish
            self.status = value
            return
            
        self.headers.append((key, value))
    
    def http_headers(self, more_headers=[]):
        for header in more_headers:
            self.setHttpHeader(header)
        
        if not self.hasContentType:
            self.headers.insert(0, ('Content-Type', 'text/html;charset=%s' % config.charset))
    
    def flush(self):
        pass
    
    def finish(self):
        pass
    
    def output(self):
        return self.stdout.getvalue()


