Changeset 5217 in subversion

Timestamp:

Sep 15, 2011 5:42:52 AM (21 months ago)

Author:

alec

Message:

• Cache synchronization with QRESYNC/CONDSTORE

Location:

branches/devel-mcache/roundcubemail

Files:

• SQL/mssql.initial.sql (modified) (1 diff)
• SQL/mysql.initial.sql (modified) (1 diff)
• SQL/postgres.initial.sql (modified) (1 diff)
• SQL/sqlite.initial.sql (modified) (1 diff)
• program/include/rcube_imap.php (modified) (2 diffs)
• program/include/rcube_imap_cache.php (modified) (25 diffs)
• program/include/rcube_imap_generic.php (modified) (8 diffs)
• program/steps/mail/check_recent.inc (modified) (1 diff)
• program/steps/mail/list.inc (modified) (2 diffs)

branches/devel-mcache/roundcubemail/SQL/mssql.initial.sql

@@ -12 +12 @@
         [mailbox] [varchar] (128) COLLATE Latin1_General_CI_AI NOT NULL ,
         [changed] [datetime] NOT NULL ,
+        [valid] [char] (1) COLLATE Latin1_General_CI_AI NOT NULL ,
         [data] [text] COLLATE Latin1_General_CI_AI NOT NULL 
 ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]

branches/devel-mcache/roundcubemail/SQL/mysql.initial.sql

@@ -56 +56 @@
  `mailbox` varchar(255) /*!40101 CHARACTER SET ascii COLLATE ascii_general_ci */ NOT NULL,
  `changed` datetime NOT NULL DEFAULT '1000-01-01 00:00:00',
+ `valid` tinyint(1) NOT NULL DEFAULT '0',
  `data` longtext NOT NULL,
  CONSTRAINT `user_id_fk_cache_index` FOREIGN KEY (`user_id`)

branches/devel-mcache/roundcubemail/SQL/postgres.initial.sql

@@ -198 +198 @@
     mailbox varchar(255) NOT NULL,
     changed timestamp with time zone DEFAULT now() NOT NULL,
+    valid smallint NOT NULL DEFAULT 0,
     data text NOT NULL,
     PRIMARY KEY (user_id, mailbox)

branches/devel-mcache/roundcubemail/SQL/sqlite.initial.sql

@@ -130 +130 @@
     mailbox varchar(255) NOT NULL,
     changed datetime NOT NULL default '0000-00-00 00:00:00',
+    valid smallint NOT NULL DEFAULT '0',
     data text NOT NULL,
     PRIMARY KEY (user_id, mailbox)

branches/devel-mcache/roundcubemail/program/include/rcube_imap.php

@@ -3400 +3400 @@
             if ($this->conn->select($mailbox))
                 $this->mailbox = $mailbox;
+            else
+                return null;
         }
 
@@ -3485 +3487 @@
 
         return $options;
+    }
+
+
+    /**
+     * Synchronizes messages cache.
+     *
+     * @param string $mailbox Folder name
+     */
+    public function mailbox_sync($mailbox)
+    {
+        if ($mcache = $this->get_mcache_engine()) {
+            $mcache->synchronize($mailbox);
+        }
+        else {
+            // @TODO: use QRESYNC/CONDSTORE to detect flag changes
+        }
     }
 

branches/devel-mcache/roundcubemail/program/include/rcube_imap_cache.php

@@ -106 +106 @@
 
         // Seek in internal cache
-        if (array_key_exists('index', $this->icache[$mailbox])
-            && ($sort_field == 'ANY' || $this->icache[$mailbox]['index']['sort_field'] == $sort_field)
-        ) {
-            if ($this->icache[$mailbox]['index']['sort_order'] == $sort_order)
-                return $this->icache[$mailbox]['index']['result'];
-            else
-                return array_reverse($this->icache[$mailbox]['index']['result'], true);
+        if (array_key_exists('index', $this->icache[$mailbox])) {
+            // The index was fetched from database already, but not validated yet
+            if (!array_key_exists('result', $this->icache[$mailbox]['index'])) {
+                $index = $this->icache[$mailbox]['index'];
+            }
+            // We've got a valid index
+            else if ($sort_field == 'ANY' || $this->icache[$mailbox]['index']['sort_field'] == $sort_field
+            ) {
+                if ($this->icache[$mailbox]['index']['sort_order'] == $sort_order)
+                    return $this->icache[$mailbox]['index']['result'];
+                else
+                    return array_reverse($this->icache[$mailbox]['index']['result'], true);
+            }
         }
 
         // Get index from DB (if DB wasn't already queried)
-        if (empty($this->icache[$mailbox]['index_queried'])) {
+        if (empty($index) && empty($this->icache[$mailbox]['index_queried'])) {
             $index = $this->get_index_row($mailbox);
 
@@ -124 +130 @@
             $this->icache[$mailbox]['index_queried'] = true;
         }
-        $data  = null;
+
+        $data = null;
 
         // @TODO: Think about skipping validation checks.
@@ -132 +139 @@
         // and many rcube_imap changes to connect when needed
 
-        // Entry exist, check cache status
+        // Entry exists, check cache status
         if (!empty($index)) {
             $exists = true;
@@ -156 +163 @@
         }
         else {
+            if ($existing) {
+                return null;
+            }
+            else if ($sort_field == 'ANY') {
+                $sort_field = '';
+            }
+
             // Got it in internal cache, so the row already exist
             $exists = array_key_exists('index', $this->icache[$mailbox]);
-
-            if ($existing) {
-                return null;
-            }
-            else if ($sort_field == 'ANY') {
-                $sort_field = '';
-            }
         }
 
@@ -171 +178 @@
             // Get mailbox data (UIDVALIDITY, counters, etc.) for status check
             $mbox_data = $this->imap->mailbox_data($mailbox);
-            $data      = array();
-
-            // Prevent infinite loop.
-            // It happens when rcube_imap::message_index_direct() is called.
-            // There id2uid() is called which will again call get_index() and so on.
-            if (!$sort_field && !$this->skip_deleted)
-                $this->icache['pending_index_update'] = true;
-
-            if ($mbox_data['EXISTS']) {
-                // fetch sorted sequence numbers
-                $data_seq = $this->imap->message_index_direct($mailbox, $sort_field, $sort_order);
-                // fetch UIDs
-                if (!empty($data_seq)) {
-                    // Seek in internal cache
-                    if (array_key_exists('index', (array)$this->icache[$mailbox]))
-                        $data_uid = $this->icache[$mailbox]['index']['result'];
-                    else
-                        $data_uid = $this->imap->conn->fetchUIDs($mailbox, $data_seq);
-
-                    // build index
-                    if (!empty($data_uid)) {
-                        foreach ($data_seq as $seq)
-                            if ($uid = $data_uid[$seq])
-                                $data[$seq] = $uid;
-                    }
-                }
-            }
-
-            // Reset internal flags
-            $this->icache['pending_index_update'] = false;
+            $data      = $this->get_index_data($mailbox, $sort_field, $sort_order, $mbox_data);
 
             // insert/update
-            $this->add_index_row($mailbox, $sort_field, $sort_order, $data, $mbox_data, $exists);
+            $this->add_index_row($mailbox, $sort_field, $sort_order, $data, $mbox_data,
+                $exists, $index['modseq']);
         }
 
@@ -210 +189 @@
             'sort_field' => $sort_field,
             'sort_order' => $sort_order,
+            'modseq'     => !empty($index['modseq']) ? $index['modseq'] : $mbox_data['HIGHESTMODSEQ']
         );
 
@@ -353 +333 @@
      * @param string $mailbox  Folder name
      * @param int    $uid      Message UID
+     * @param bool   $update   If message doesn't exists in cache it will be fetched
+     *                         from IMAP server
+     * @param bool   $no_cache Enables internal cache usage
      *
      * @return rcube_mail_header Message data
      */
-    function get_message($mailbox, $uid)
+    function get_message($mailbox, $uid, $update = true, $cache = true)
     {
         // Check internal cache
@@ -383 +366 @@
 
         // Get the message from IMAP server
-        if (empty($message)) {
+        if (empty($message) && $update) {
             $message = $this->imap->get_headers($uid, $mailbox, true);
             // cache will be updated in close(), see below
@@ -394 +377 @@
         // - set \Seen flag (UPDATE)
         // This way we can skip one UPDATE
-        if (!empty($message)) {
+        if (!empty($message) && $cache) {
             // Save current message from internal cache
             $this->save_icache();
@@ -534 +517 @@
      *
      * @param string  $mailbox     Folder name
-     */
-    function remove_index($mailbox = null)
-    {
-        $this->db->query(
-            "DELETE FROM ".get_table_name('cache_index')
-            ." WHERE user_id = ".intval($this->userid)
-                .(strlen($mailbox) ? " AND mailbox = ".$this->db->quote($mailbox) : "")
-        );
+     * @param bool    $remove      Enable to remove the DB row
+     */
+    function remove_index($mailbox = null, $remove = false)
+    {
+        // The index should be only removed from database when
+        // UIDVALIDITY was detected or the mailbox is empty
+        // otherwise use 'valid' flag to not loose HIGHESTMODSEQ value
+        if ($remove)
+            $this->db->query(
+                "DELETE FROM ".get_table_name('cache_index')
+                ." WHERE user_id = ".intval($this->userid)
+                    .(strlen($mailbox) ? " AND mailbox = ".$this->db->quote($mailbox) : "")
+            );
+        else
+            $this->db->query(
+                "UPDATE ".get_table_name('cache_index')
+                ." SET valid = 0"
+                ." WHERE user_id = ".intval($this->userid)
+                    .(strlen($mailbox) ? " AND mailbox = ".$this->db->quote($mailbox) : "")
+            );
 
         if (strlen($mailbox))
@@ -578 +573 @@
     function clear($mailbox = null, $uids = null)
     {
-        $this->remove_index($mailbox);
+        $this->remove_index($mailbox, true);
         $this->remove_thread($mailbox);
         $this->remove_message($mailbox, $uids);
@@ -619 +614 @@
     }
 
-
     /**
      * Fetches index data from database
@@ -627 +621 @@
         // Get index from DB
         $sql_result = $this->db->query(
-            "SELECT data"
+            "SELECT data, valid"
             ." FROM ".get_table_name('cache_index')
             ." WHERE user_id = ?"
@@ -637 +631 @@
 
             return array(
+                'valid'      => $sql_arr['valid'],
                 'seq'        => explode(',', $data[0]),
                 'uid'        => explode(',', $data[1]),
@@ -644 +639 @@
                 'validity'   => $data[5],
                 'uidnext'    => $data[6],
+                'modseq'     => $data[7],
             );
         }
@@ -690 +686 @@
      */
     private function add_index_row($mailbox, $sort_field, $sort_order,
-        $data = array(), $mbox_data = array(), $exists = false)
+        $data = array(), $mbox_data = array(), $exists = false, $modseq = null)
     {
         $data = array(
@@ -700 +696 @@
             (int) $mbox_data['UIDVALIDITY'],
             (int) $mbox_data['UIDNEXT'],
+            $modseq ? $modseq : $mbox_data['HIGHESTMODSEQ'],
         );
         $data = implode('@', $data);
@@ -706 +703 @@
             $sql_result = $this->db->query(
                 "UPDATE ".get_table_name('cache_index')
-                ." SET data = ?, changed = ".$this->db->now()
+                ." SET data = ?, valid = 1, changed = ".$this->db->now()
                 ." WHERE user_id = ?"
                     ." AND mailbox = ?",
@@ -713 +710 @@
             $sql_result = $this->db->query(
                 "INSERT INTO ".get_table_name('cache_index')
-                ." (user_id, mailbox, data, changed)"
-                ." VALUES (?, ?, ?, ".$this->db->now().")",
+                ." (user_id, mailbox, data, valid, changed)"
+                ." VALUES (?, ?, ?, 1, ".$this->db->now().")",
                 $this->userid, $mailbox, $data);
     }
@@ -765 +762 @@
 
         // Check UIDVALIDITY
-        // @TODO: while we're storing message sequence numbers in thread
-        //        index, should UIDVALIDITY invalidate the thread data?
         if ($index['validity'] != $mbox_data['UIDVALIDITY']) {
-            // the whole cache (all folders) is invalid
-            $this->clear();
+            $this->clear($mailbox);
             $exists = false;
             return false;
@@ -781 +775 @@
         }
 
+        // Validation flag
+        if (!$is_thread && empty($index['valid'])) {
+            unset($this->icache[$mailbox][$is_thread ? 'thread' : 'index']);
+            return false;
+        }
+
+        // Index was created with different skip_deleted setting
+        if ($this->skip_deleted != $index['deleted']) {
+            return false;
+        }
+
+        // Check HIGHESTMODSEQ
+        if (!empty($index['modseq']) && !empty($mbox_data['HIGHESTMODSEQ'])
+            && $index['modseq'] == $mbox_data['HIGHESTMODSEQ']
+        ) {
+            return true;
+        }
+
         // Check UIDNEXT
         if ($index['uidnext'] != $mbox_data['UIDNEXT']) {
             unset($this->icache[$mailbox][$is_thread ? 'thread' : 'index']);
-            return false;
-        }
-
-        // Index was created with different skip_deleted setting
-        if ($this->skip_deleted != $index['deleted']) {
             return false;
         }
@@ -849 +856 @@
 
     /**
+     * Synchronizes the mailbox.
+     *
+     * @param string $mailbox Folder name
+     */
+    function synchronize($mailbox)
+    {
+        // RFC4549: Synchronization Operations for Disconnected IMAP4 Clients
+        // RFC4551: IMAP Extension for Conditional STORE Operation
+        //          or Quick Flag Changes Resynchronization
+        // RFC5162: IMAP Extensions for Quick Mailbox Resynchronization
+
+        // @TODO: synchronize with other methods?
+        $qresync   = $this->imap->get_capability('QRESYNC');
+        $condstore = $qresync ? true : $this->imap->get_capability('CONDSTORE');
+
+        if (!$qresync && !$condstore) {
+            return;
+        }
+
+        // Get stored index
+        $index = $this->get_index_row($mailbox);
+
+        // database is empty
+        if (empty($index)) {
+            // set the flag that DB was already queried for index
+            // this way we'll be able to skip one SELECT in get_index()
+            $this->icache[$mailbox]['index_queried'] = true;
+            return;
+        }
+
+        $this->icache[$mailbox]['index'] = $index;
+
+        // no last HIGHESTMODSEQ value
+        if (empty($index['modseq'])) {
+            return;
+        }
+
+        // NOTE: make sure the mailbox isn't selected, before
+        // enabling QRESYNC and invoking SELECT
+        if ($this->imap->conn->selected !== null) {
+            $this->imap->conn->close();
+        }
+
+        // Enable QRESYNC
+        $res = $this->imap->conn->enable($qresync ? 'QRESYNC' : 'CONDSTORE');
+        if (!is_array($res)) {
+            return;
+        }
+
+        // Get mailbox data (UIDVALIDITY, HIGHESTMODSEQ, counters, etc.)
+        $mbox_data = $this->imap->mailbox_data($mailbox);
+
+        if (empty($mbox_data)) {
+             return;
+        }
+
+        // Check UIDVALIDITY
+        if ($index['validity'] != $mbox_data['UIDVALIDITY']) {
+            $this->clear($mailbox);
+            return;
+        }
+
+        // QRESYNC not supported on specified mailbox
+        if (!empty($mbox_data['NOMODSEQ']) || empty($mbox_data['HIGHESTMODSEQ'])) {
+            return;
+        }
+
+        // Nothing new
+        if ($mbox_data['HIGHESTMODSEQ'] == $index['modseq']) {
+            return;
+        }
+
+        // Get known uids
+        $uids = array();
+        $sql_result = $this->db->query(
+            "SELECT uid"
+            ." FROM ".get_table_name('cache_messages')
+            ." WHERE user_id = ?"
+                ." AND mailbox = ?",
+            $this->userid, $mailbox);
+
+        while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
+          $uids[] = $sql_arr['uid'];
+        }
+
+        // No messages in database, nothing to sync
+        if (empty($uids)) {
+            return;
+        }
+
+        $flags = $this->flag_fields;
+
+        // Get modified flags and vanished messages
+        // UID FETCH 1:* (FLAGS) (CHANGEDSINCE 0123456789 VANISHED)
+        $result = $this->imap->conn->fetch($mailbox,
+            !empty($uids) ? $uids : '1:*', true, array('FLAGS'),
+            $index['modseq'], $qresync);
+
+        if (!empty($result)) {
+            foreach ($result as $id => $msg) {
+                $uid = $msg->uid;
+                // Remove deleted message
+                if ($this->skip_deleted && $msg->deleted) {
+                    $this->remove_message($mailbox, $uid);
+                    continue;
+                }
+
+                // Get the message to merge the flags (if it exists)
+                // @TODO: update flags without fetching the cached message
+                $message = $this->get_message($mailbox, $uid, false, false);
+
+                if (empty($message))
+                    continue;
+
+                $message->id = $id;
+
+                // reset message flags
+                $message->flags = $msg->flags;
+                foreach ($flags as $flag)
+                    $message->$flag = $msg->$flag;
+
+                $this->add_message($mailbox, $message);
+            }
+        }
+
+        // Get VANISHED
+        if ($qresync) {
+            $mbox_data = $this->imap->mailbox_data($mailbox);
+
+            // Removed messages
+            if (!empty($mbox_data['VANISHED'])) {
+                $uids = rcube_imap_generic::uncompressMessageSet($mbox_data['VANISHED']);
+                if (!empty($uids)) {
+                    // remove messages from database
+                    $this->remove_message($mailbox, $uids);
+
+                    // Invalidate thread indexes (?)
+                    $this->remove_thread($mailbox);
+                }
+            }
+        }
+
+        $sort_field = $index['sort_field'];
+        $sort_order = $index['sort_order'];
+        $exists     = true;
+
+        // Validate index
+        if (!$this->validate($mailbox, $index, $exists)) {
+            // Update index
+            $data = $this->get_index_data($mailbox, $sort_field, $sort_order, $mbox_data);
+        }
+        else {
+            $data = array_combine($index['seq'], $index['uid']);
+        }
+
+        // update index and/or HIGHESTMODSEQ value
+        $this->add_index_row($mailbox, $sort_field, $sort_order, $data, $mbox_data, $exists);
+
+        // update internal cache for get_index()
+        $this->icache[$mailbox]['index']['result'] = $data;
+    }
+
+
+    /**
      * Converts cache row into message object.
      *
@@ -907 +1078 @@
      * Prepares message object to be stored in database.
      */
-    private function message_object_prepare($msg, $recursive = false)
-    {
-        // Remove body too big (>500kB)
-        if ($recursive || ($msg->body && strlen($msg->body) > 500 * 1024)) {
+    private function message_object_prepare($msg)
+    {
+        // Remove body too big (>25kB)
+        if ($msg->body && strlen($msg->body) > 25 * 1024) {
             unset($msg->body);
         }
@@ -923 +1094 @@
         if (is_array($msg->structure->parts)) {
             foreach ($msg->structure->parts as $idx => $part) {
-                $msg->structure->parts[$idx] = $this->message_object_prepare($part, true);
+                $msg->structure->parts[$idx] = $this->message_object_prepare($part);
             }
         }
@@ -929 +1100 @@
         return $msg;
     }
+
+
+    /**
+     * Fetches index data from IMAP server
+     */
+    private function get_index_data($mailbox, $sort_field, $sort_order, $mbox_data = array())
+    {
+        $data      = array();
+
+        if (empty($mbox_data)) {
+            $mbox_data = $this->imap->mailbox_data($mailbox);
+        }
+
+        // Prevent infinite loop.
+        // It happens when rcube_imap::message_index_direct() is called.
+        // There id2uid() is called which will again call get_index() and so on.
+        if (!$sort_field && !$this->skip_deleted)
+            $this->icache['pending_index_update'] = true;
+
+        if ($mbox_data['EXISTS']) {
+            // fetch sorted sequence numbers
+            $data_seq = $this->imap->message_index_direct($mailbox, $sort_field, $sort_order);
+            // fetch UIDs
+            if (!empty($data_seq)) {
+                // Seek in internal cache
+                if (array_key_exists('index', (array)$this->icache[$mailbox])
+                    && array_key_exists('result', (array)$this->icache[$mailbox]['index'])
+                )
+                    $data_uid = $this->icache[$mailbox]['index']['result'];
+                else
+                    $data_uid = $this->imap->conn->fetchUIDs($mailbox, $data_seq);
+
+                // build index
+                if (!empty($data_uid)) {
+                    foreach ($data_seq as $seq)
+                        if ($uid = $data_uid[$seq])
+                            $data[$seq] = $uid;
+                }
+            }
+        }
+
+        // Reset internal flags
+        $this->icache['pending_index_update'] = false;
+
+        return $data;
+    }
 }

branches/devel-mcache/roundcubemail/program/include/rcube_imap_generic.php

@@ -689 +689 @@
         $this->error    = '';
         $this->errornum = self::ERROR_OK;
-        $this->selected = '';
+        $this->selected = null;
         $this->user     = $user;
         $this->host     = $host;
@@ -881 +881 @@
         }
 
-        if ($this->selected == $mailbox) {
+        if ($this->selected === $mailbox) {
             return true;
         }
@@ -1044 +1044 @@
 
         if ($result == self::ERROR_OK) {
-            $this->selected = ''; // state has changed, need to reselect
+            $this->selected = null; // state has changed, need to reselect
             return true;
         }
@@ -1062 +1062 @@
 
         if ($result == self::ERROR_OK) {
-            $this->selected = '';
+            $this->selected = null;
             return true;
         }
@@ -1129 +1129 @@
 
         if ($res) {
-            if ($this->selected == $mailbox)
+            if ($this->selected === $mailbox)
                 $res = $this->close();
             else
@@ -1148 +1148 @@
     {
         if ($refresh) {
-            $this->selected = '';
-        }
-
-        if ($this->selected == $mailbox) {
+            $this->selected = null;
+        }
+
+        if ($this->selected === $mailbox) {
             return $this->data['EXISTS'];
         }
@@ -1185 +1185 @@
         $this->select($mailbox);
 
-        if ($this->selected == $mailbox) {
+        if ($this->selected === $mailbox) {
             return $this->data['RECENT'];
         }
@@ -1807 +1807 @@
             // Sample: * VANISHED (EARLIER) 300:310,405,411
 
-            else if (preg_match('/^\* VANISHED [EARLIER]*/i', $line, $match)) {
+            else if (preg_match('/^\* VANISHED [()EARLIER]*/i', $line, $match)) {
                 $line   = substr($line, strlen($match[0]));
                 $v_data = $this->tokenizeResponse($line, 1);

branches/devel-mcache/roundcubemail/program/steps/mail/check_recent.inc

@@ -35 +35 @@
 // check recent/unseen counts
 foreach ($a_mailboxes as $mbox_name) {
-    if ($mbox_name == $current && ($status = $IMAP->mailbox_status($mbox_name))) {
+    if ($mbox_name == $current) {
+        // Synchronize mailbox cache, handle flag changes
+        $IMAP->mailbox_sync($mbox_name);
 
+        // Get mailbox status
+        $status = $IMAP->mailbox_status($mbox_name);
+    }
+    else {
+        $status = null;
+    }
+
+    if ($status) {
         rcmail_send_unread_count($mbox_name, true);
 

branches/devel-mcache/roundcubemail/program/steps/mail/list.inc

@@ -53 +53 @@
 
 $mbox_name = $IMAP->get_mailbox_name();
+
+// Synchronize mailbox cache, handle flag changes
+$IMAP->mailbox_sync($mbox_name);
 
 // initialize searching result if search_filter is used
@@ -117 +120 @@
 // send response
 $OUTPUT->send();
-
-