"SfR Fresh" - the SfR Freeware/Shareware Archive

Member "pan-0.133/pan/data/article-cache.cc" of archive pan-0.133.tar.gz:

    1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
    2 /*
    3  * Pan - A Newsreader for Gtk+
    4  * Copyright (C) 2002-2006  Charles Kerr <charles@rebelbase.com>
    5  *
    6  * This program is free software; you can redistribute it and/or modify
    7  * it under the terms of the GNU General Public License as published by
    8  * the Free Software Foundation; version 2 of the License.
    9  *
   10  * This program is distributed in the hope that it will be useful,
   11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   13  * GNU General Public License for more details.
   14  *
   15  * You should have received a copy of the GNU General Public License
   16  * along with this program; if not, write to the Free Software
   17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
   18  */
   19 
   20 #include <config.h>
   21 
   22 extern "C"
   23 {
   24   #include <errno.h>
   25   #include <sys/types.h>
   26   #include <sys/stat.h>
   27   #include <unistd.h>
   28   #include <dirent.h>
   29 
   30   #include <glib.h>
   31   #include <glib/gi18n.h>
   32   #include <gmime/gmime.h>
   33 }
   34 
   35 #include <pan/general/debug.h>
   36 #include <pan/general/file-util.h>
   37 #include <pan/general/macros.h>
   38 #include <pan/general/messages.h>
   39 #include <pan/general/log.h>
   40 #include <pan/general/string-view.h>
   41 #include <pan/usenet-utils/mime-utils.h>
   42 #include "article.h"
   43 #include "article-cache.h"
   44 
   45 using namespace pan;
   46 
   47 /*****
   48 ******
   49 *****/
   50 
   51 namespace
   52 {
   53    /**
   54     * Some characters in message-ids don't work well in filenames,
   55     * so we transform them to a safer name.
   56     */
   57    char*
   58    message_id_to_filename (char * buf, int len, const StringView& mid)
   59    {
   60       // sanity clause
   61       pan_return_val_if_fail (!mid.empty(), 0);
   62       pan_return_val_if_fail (buf!=0, NULL);
   63       pan_return_val_if_fail (len>0, NULL);
   64 
   65       // some characters in message-ids are illegal on older Windows boxes,
   66       // so we transform those illegal characters using URL encoding
   67       char * out = buf;
   68       for (const char *in=mid.begin(), *end=mid.end(); in!=end; ++in) {
   69          switch (*in) {
   70             case '%': /* this is the escape character */
   71             case '"': case '*': case '/': case ':': case '?': case '|':
   72             case '\\': /* these are illegal on vfat, fat32 */
   73                g_snprintf (out, len-(out-buf), "%%%02x", (int)*in);
   74                out += 3;
   75                break;
   76             case '<': case '>': /* these are illegal too, but rather than encoding
   77                                    them, follow the convention of omitting them */
   78                break;
   79             default:
   80                *out++ = *in;
   81          }
   82       }
   83 
   84       g_snprintf (out, len-(out-buf), ".msg");
   85       return buf;
   86    }
   87 
   88    /**
   89     * Message-IDs are transformed via message_id_to_filename()
   90     * to play nicely with some filesystems, so to extract the Message-ID
   91     * from a filename we need to reverse the transform.
   92     *
   93     * @return string length, or 0 on failure
   94     */
   95    int
   96    filename_to_message_id (char * buf, int len, const char * basename)
   97    {
   98       const char * in;
   99       char * out;
  100       char * pch;
  101       char tmp_basename[PATH_MAX];
  102 
  103       // sanity clause
  104       pan_return_val_if_fail (basename && *basename, 0);
  105       pan_return_val_if_fail (buf!=NULL, 0);
  106       pan_return_val_if_fail (len>0, 0);
  107 
  108       // remove the trailing ".msg"
  109       g_strlcpy (tmp_basename, basename, sizeof(tmp_basename));
  110       if ((pch = g_strrstr (tmp_basename, ".msg")))
  111          *pch = '\0';
  112       g_strstrip (tmp_basename);
  113 
  114       // transform
  115       out = buf;
  116       *out++ = '<';
  117       for (in=tmp_basename; *in; ++in) {
  118          if (in[0]!='%' || !g_ascii_isxdigit(in[1]) || !g_ascii_isxdigit(in[2]))
  119             *out++ = *in;
  120          else {
  121             char buf[3];
  122             buf[0] = *++in;
  123             buf[1] = *++in;
  124             buf[2] = '\0';
  125             *out++ = (char) strtoul (buf, NULL, 16);
  126          }
  127       }
  128       *out++ = '>';
  129       *out = '\0';
  130 
  131       return out - buf;
  132    }
  133 };
  134 
  135 /*****
  136 ******
  137 *****/
  138 
  139 ArticleCache :: ArticleCache (const StringView& path, size_t max_megs):
  140    _path (path.str, path.len),
  141    _max_megs (max_megs),
  142    _current_bytes (0ul)
  143 {
  144    GError * err = NULL;
  145    GDir * dir = g_dir_open (_path.c_str(), 0, &err);
  146    if (err != NULL)
  147    {
  148       Log::add_err_va (_("Error opening directory: \"%s\": %s"), _path.c_str(), err->message);
  149       g_clear_error (&err);
  150    }
  151    else
  152    {
  153       char filename[PATH_MAX];
  154       const char * fname;
  155       while ((fname = g_dir_read_name (dir)))
  156       {
  157          struct stat stat_p;
  158          g_snprintf (filename, sizeof(filename), "%s%c%s", _path.c_str(), G_DIR_SEPARATOR, fname);
  159          if (!stat (filename, &stat_p))
  160          {
  161             char str[2048];
  162             const int len (filename_to_message_id (str, sizeof(str), fname));
  163             if (len != 0)
  164             {
  165                MsgInfo info;
  166                info._message_id = StringView (str, len);
  167                info._size = stat_p.st_size;
  168                info._date = stat_p.st_mtime;
  169                _current_bytes += info._size;
  170                _mid_to_info.insert (mid_to_info_t::value_type (info._message_id, info));
  171             }
  172          }
  173       }
  174       g_dir_close (dir);
  175       debug ("loaded " << _mid_to_info.size() << " articles into cache from " << _path);
  176    }
  177 }
  178 
  179 ArticleCache :: ~ArticleCache ()
  180 {
  181 }
  182 
  183 /*****
  184 ******
  185 *****/
  186 
  187 void
  188 ArticleCache :: fire_added (const Quark& mid)
  189 {
  190   for (listeners_t::iterator it(_listeners.begin()), end(_listeners.end()); it!=end; )
  191     (*it++)->on_cache_added (mid);
  192 }
  193 
  194 void
  195 ArticleCache :: fire_removed (const quarks_t& mids)
  196 {
  197   for (listeners_t::iterator it(_listeners.begin()), end(_listeners.end()); it!=end; )
  198     (*it++)->on_cache_removed (mids);
  199 }
  200 
  201 /*****
  202 ******
  203 *****/
  204 
  205 bool
  206 ArticleCache :: contains (const Quark& mid) const
  207 {
  208   return _mid_to_info.find (mid) != _mid_to_info.end();
  209 }
  210 
  211 char*
  212 ArticleCache :: get_filename (char * buf, int buflen, const Quark& mid) const
  213 {
  214    char basename[PATH_MAX];
  215    *buf = '\0';
  216    message_id_to_filename (basename, sizeof(basename), mid.to_view());
  217    g_snprintf (buf, buflen, "%s%c%s", _path.c_str(), G_DIR_SEPARATOR, basename);
  218    return buf && *buf ? buf : 0;
  219 };
  220 
  221 bool
  222 ArticleCache :: add (const Quark& message_id, const StringView& article)
  223 {
  224   debug ("adding " << message_id << ", which is " << article.len << " bytes long");
  225 
  226   pan_return_val_if_fail (!message_id.empty(), false);
  227   pan_return_val_if_fail (!article.empty(), false);
  228 
  229   FILE * fp = 0;
  230   char filename[PATH_MAX];
  231   if (get_filename (filename, sizeof(filename), message_id))
  232     fp = fopen (filename, "wb+");
  233 
  234   if (!fp)
  235   {
  236       Log::add_err_va (_("Unable to save \"%s\" %s"),
  237                        filename, file::pan_strerror(errno));
  238       return false;
  239   }
  240   else
  241   {
  242     const size_t bytes_written (fwrite (article.str, sizeof(char), article.len, fp));
  243     fclose (fp);
  244 
  245     if (bytes_written < article.len)
  246     {
  247       Log::add_err_va (_("Unable to save \"%s\" %s"),
  248                        filename, file::pan_strerror(errno));
  249       return false;
  250     }
  251     else
  252     {
  253       MsgInfo info;
  254       info._message_id = message_id;
  255       info._size = article.len;
  256       info._date = time(0);
  257       _mid_to_info.insert (mid_to_info_t::value_type (info._message_id, info));
  258       fire_added (message_id);
  259 
  260       _current_bytes += info._size;
  261       resize ();
  262     }
  263   }
  264 
  265   return true;
  266 }
  267 
  268 /***
  269 ****
  270 ***/
  271 
  272 void
  273 ArticleCache :: reserve (const mid_sequence_t& mids)
  274 {
  275   foreach_const (mid_sequence_t, mids, it)
  276     ++_locks[*it];
  277 }
  278 
  279 void
  280 ArticleCache :: release (const mid_sequence_t& mids)
  281 {
  282   foreach_const (mid_sequence_t, mids, it)
  283     if (!--_locks[*it])
  284       _locks.erase (*it);
  285 }
  286 
  287 /***
  288 ****
  289 ***/
  290 
  291 void
  292 ArticleCache :: resize ()
  293 {
  294   // let's shrink it to 80% of the maximum size
  295   const double buffer_zone (0.8);
  296   guint64 max_bytes (_max_megs * 1024 * 1024);
  297   max_bytes = (guint64) ((double)max_bytes * buffer_zone);
  298   resize (max_bytes);
  299 }
  300 
  301 void
  302 ArticleCache :: clear ()
  303 {
  304   resize (0);
  305 }
  306 
  307 void
  308 ArticleCache :: resize (guint64 max_bytes)
  309 {
  310   quarks_t removed;
  311   if (_current_bytes > max_bytes)
  312   {
  313     // sort from oldest to youngest
  314     typedef std::set<MsgInfo, MsgInfoCompare> sorted_info_t;
  315     sorted_info_t si;
  316     for (mid_to_info_t::const_iterator it=_mid_to_info.begin(), end=_mid_to_info.end(); it!=end; ++it)
  317       si.insert (it->second);
  318 
  319     // start blowing away files
  320     for (sorted_info_t::const_iterator it=si.begin(), end=si.end(); _current_bytes>max_bytes && it!=end; ++it) {
  321       const Quark& mid (it->_message_id);
  322       if (_locks.find(mid) == _locks.end()) {
  323         char buf[PATH_MAX];
  324         get_filename (buf, sizeof(buf), mid);
  325         unlink (buf);
  326         _current_bytes -= it->_size;
  327         removed.insert (mid);
  328         debug ("removing [" << mid << "] as we resize the queue");
  329         _mid_to_info.erase (mid);
  330       }
  331     }
  332   }
  333 
  334   debug ("cache expired " << removed.size() << " articles, "
  335          "has " << _mid_to_info.size() << " active "
  336          "and " << _locks.size() << " locked.");
  337 
  338   if (!removed.empty())
  339     fire_removed (removed);
  340 }
  341 
  342 /****
  343 *****
  344 *****  Getting Messages
  345 *****
  346 ****/
  347 
  348 /*private*/ GMimeStream*
  349 ArticleCache :: get_message_file_stream (const Quark& mid) const
  350 {
  351    GMimeStream * retval = NULL;
  352 
  353    /* open the file */
  354    char filename[PATH_MAX];
  355    if (get_filename (filename, sizeof(filename), mid))
  356    {
  357       errno = 0;
  358       FILE * fp = fopen (filename, "rb");
  359       if (!fp)
  360          Log::add_err_va (_("Error opening file \"%s\" %s"), filename, file::pan_strerror(errno));
  361       else {
  362          GMimeStream * file_stream = g_mime_stream_file_new (fp);
  363          retval = g_mime_stream_buffer_new (file_stream, GMIME_STREAM_BUFFER_BLOCK_READ);
  364          g_object_unref (file_stream);
  365       }
  366    }
  367 
  368    debug ("file stream for " << mid << ": " << retval);
  369    return retval;
  370 }
  371 
  372 /*private*/ GMimeStream*
  373 ArticleCache :: get_message_mem_stream (const Quark& mid) const
  374 {
  375    debug ("mem stream got quark " << mid);
  376    GMimeStream * retval (0);
  377 
  378    char filename[PATH_MAX];
  379    if (get_filename (filename, sizeof(filename), mid))
  380    {
  381       debug ("mem stream loading filename " << filename);
  382       gsize len (0);
  383       char * buf (0);
  384       GError * err (0);
  385 
  386       if (g_file_get_contents (filename, &buf, &len, &err)) {
  387          debug ("got the contents, calling mem_new_with_buffer");
  388          retval = g_mime_stream_mem_new_with_buffer (buf, len);
  389          g_free (buf);
  390       } else {
  391          Log::add_err_va (_("Error reading file \"%s\": %s"), filename, err->message);
  392          g_clear_error (&err);
  393       }
  394    }
  395 
  396    debug ("mem stream for " << mid << ": " << retval);
  397    return retval;
  398 }
  399 
  400 GMimeMessage*
  401 ArticleCache :: get_message (const mid_sequence_t& mids)
  402 {
  403    debug ("trying to get a message with " << mids.size() << " parts");
  404    GMimeMessage * retval = NULL;
  405 
  406    // load the streams
  407    typedef std::vector<GMimeStream*> streams_t;
  408    streams_t streams;
  409    //const bool in_memory (mids.size() <= 2u);
  410    const bool in_memory (true); // workaround for bug #439841
  411    foreach_const (mid_sequence_t, mids, it) {
  412       const Quark mid (*it);
  413       GMimeStream * stream (0);
  414       if (this->contains (*it))
  415         stream = in_memory
  416           ? get_message_mem_stream (mid)
  417           : get_message_file_stream (mid);
  418       if (stream)
  419         streams.push_back (stream);
  420    }
  421 
  422    // build the message
  423    if (!streams.empty())
  424      retval = mime :: construct_message (&streams.front(), streams.size());
  425 
  426    // cleanup
  427    foreach (streams_t, streams, it)
  428      g_object_unref (*it);
  429 
  430    debug ("returning " << retval);
  431    return retval;
  432 }
  433 
  434 ArticleCache :: strings_t
  435 ArticleCache :: get_filenames (const mid_sequence_t& mids)
  436 {
  437   strings_t ret;
  438   char filename[PATH_MAX];
  439   foreach_const (mid_sequence_t, mids, it)
  440     if (get_filename (filename, sizeof(filename), *it))
  441       ret.push_back (filename);
  442   return ret;
  443 }