kmail

kmfolderindex.cpp

00001 /* -*- mode: C++; c-file-style: "gnu" -*-
00002     This file is part of KMail, the KDE mail client.
00003     Copyright (c) 2000 Don Sanders <sanders@kde.org>
00004 
00005     KMail is free software; you can redistribute it and/or modify it
00006     under the terms of the GNU General Public License, version 2, as
00007     published by the Free Software Foundation.
00008 
00009     KMail is distributed in the hope that it will be useful, but
00010     WITHOUT ANY WARRANTY; without even the implied warranty of
00011     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00012     General Public License for more details.
00013 
00014     You should have received a copy of the GNU General Public License
00015     along with this program; if not, write to the Free Software
00016     Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
00017 */
00018 
00019 #include "kmfolderindex.h"
00020 #include "kmfolder.h"
00021 #include "kmfoldertype.h"
00022 #include <config.h>
00023 #include <qfileinfo.h>
00024 #include <qtimer.h>
00025 #include <kdebug.h>
00026 
00027 
00028 #define HAVE_MMAP //need to get this into autoconf FIXME  --Sam
00029 #include <unistd.h>
00030 #ifdef HAVE_MMAP
00031 #include <sys/mman.h>
00032 #endif
00033 
00034 // Current version of the table of contents (index) files
00035 #define INDEX_VERSION 1507
00036 
00037 #ifndef MAX_LINE
00038 #define MAX_LINE 4096
00039 #endif
00040 
00041 #ifndef INIT_MSGS
00042 #define INIT_MSGS 8
00043 #endif
00044 
00045 #include <errno.h>
00046 #include <assert.h>
00047 #include <utime.h>
00048 #include <fcntl.h>
00049 
00050 #ifdef HAVE_BYTESWAP_H
00051 #include <byteswap.h>
00052 #endif
00053 #include <kapplication.h>
00054 #include <kcursor.h>
00055 #include <kmessagebox.h>
00056 #include <klocale.h>
00057 #include "kmmsgdict.h"
00058 
00059 // We define functions as kmail_swap_NN so that we don't get compile errors
00060 // on platforms where bswap_NN happens to be a function instead of a define.
00061 
00062 /* Swap bytes in 32 bit value.  */
00063 #ifdef bswap_32
00064 #define kmail_swap_32(x) bswap_32(x)
00065 #else
00066 #define kmail_swap_32(x) \
00067      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) |           \
00068       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
00069 #endif
00070 
00071 #include <stdlib.h>
00072 #include <sys/types.h>
00073 #include <sys/stat.h>
00074 #include <sys/file.h>
00075 
00076 KMFolderIndex::KMFolderIndex(KMFolder* folder, const char* name)
00077   : FolderStorage(folder, name), mMsgList(INIT_MSGS)
00078 {
00079     mIndexStream = 0;
00080     mIndexStreamPtr = 0;
00081     mIndexStreamPtrLength = 0;
00082     mIndexSwapByteOrder = false;
00083     mIndexSizeOfLong = sizeof(long);
00084     mIndexId = 0;
00085     mHeaderOffset   = 0;
00086 }
00087 
00088 
00089 KMFolderIndex::~KMFolderIndex()
00090 {
00091 }
00092 
00093 
00094 QString KMFolderIndex::indexLocation() const
00095 {
00096   QString sLocation(folder()->path());
00097 
00098   if ( !sLocation.isEmpty() ) {
00099     sLocation += '/';
00100     sLocation += '.';
00101   }
00102   sLocation += dotEscape(fileName());
00103   sLocation += ".index";
00104 
00105   return sLocation;
00106 }
00107 
00108 int KMFolderIndex::updateIndex()
00109 {
00110   if (!mAutoCreateIndex)
00111     return 0;
00112   bool dirty = mDirty;
00113   mDirtyTimer->stop();
00114   for ( unsigned int i = 0; !dirty && i < mMsgList.high(); i++ ) {
00115     if ( mMsgList.at(i) ) {
00116       if ( !mMsgList.at(i)->syncIndexString() ) {
00117         dirty = true;
00118       }
00119     }
00120   }
00121   if (!dirty) { // Update successful
00122       touchFolderIdsFile();
00123       return 0;
00124   }
00125   return writeIndex();
00126 }
00127 
00128 int KMFolderIndex::writeIndex( bool createEmptyIndex )
00129 {
00130   QString tempName;
00131   QString indexName;
00132   mode_t old_umask;
00133   int len;
00134   const uchar *buffer = 0;
00135 
00136   indexName = indexLocation();
00137   tempName = indexName + ".temp";
00138   unlink(QFile::encodeName(tempName));
00139 
00140   // We touch the folder, otherwise the index is regenerated, if KMail is
00141   // running, while the clock switches from daylight savings time to normal time
00142   utime(QFile::encodeName(location()), 0);
00143 
00144   old_umask = umask(077);
00145   FILE *tmpIndexStream = fopen(QFile::encodeName(tempName), "w");
00146   umask(old_umask);
00147   if (!tmpIndexStream)
00148     return errno;
00149 
00150   fprintf(tmpIndexStream, "# KMail-Index V%d\n", INDEX_VERSION);
00151 
00152   // Header
00153   Q_UINT32 byteOrder = 0x12345678;
00154   Q_UINT32 sizeOfLong = sizeof(long);
00155 
00156   Q_UINT32 header_length = sizeof(byteOrder)+sizeof(sizeOfLong);
00157   char pad_char = '\0';
00158   fwrite(&pad_char, sizeof(pad_char), 1, tmpIndexStream);
00159   fwrite(&header_length, sizeof(header_length), 1, tmpIndexStream);
00160 
00161   // Write header
00162   fwrite(&byteOrder, sizeof(byteOrder), 1, tmpIndexStream);
00163   fwrite(&sizeOfLong, sizeof(sizeOfLong), 1, tmpIndexStream);
00164 
00165   off_t nho = ftell(tmpIndexStream);
00166 
00167   if ( !createEmptyIndex ) {
00168     KMMsgBase* msgBase;
00169     for (unsigned int i=0; i<mMsgList.high(); i++)
00170     {
00171       if (!(msgBase = mMsgList.at(i))) continue;
00172       buffer = msgBase->asIndexString(len);
00173       fwrite(&len,sizeof(len), 1, tmpIndexStream);
00174 
00175       off_t tmp = ftell(tmpIndexStream);
00176       msgBase->setIndexOffset(tmp);
00177       msgBase->setIndexLength(len);
00178       if(fwrite(buffer, len, 1, tmpIndexStream) != 1)
00179     kdDebug(5006) << "Whoa! " << __FILE__ << ":" << __LINE__ << endl;
00180     }
00181   }
00182 
00183   int fError = ferror( tmpIndexStream );
00184   if( fError != 0 ) {
00185     fclose( tmpIndexStream );
00186     return fError;
00187   }
00188   if(    ( fflush( tmpIndexStream ) != 0 )
00189       || ( fsync( fileno( tmpIndexStream ) ) != 0 ) ) {
00190     int errNo = errno;
00191     fclose( tmpIndexStream );
00192     return errNo;
00193   }
00194   if( fclose( tmpIndexStream ) != 0 )
00195     return errno;
00196 
00197   ::rename(QFile::encodeName(tempName), QFile::encodeName(indexName));
00198   mHeaderOffset = nho;
00199   if (mIndexStream)
00200       fclose(mIndexStream);
00201 
00202   if ( createEmptyIndex )
00203     return 0;
00204 
00205   mIndexStream = fopen(QFile::encodeName(indexName), "r+"); // index file
00206   assert( mIndexStream );
00207   fcntl(fileno(mIndexStream), F_SETFD, FD_CLOEXEC);
00208 
00209   updateIndexStreamPtr();
00210 
00211   writeFolderIdsFile();
00212 
00213   setDirty( false );
00214   return 0;
00215 }
00216 
00217 bool KMFolderIndex::readIndex()
00218 {
00219   if ( contentsType() != KMail::ContentsTypeMail ) {
00220     kdDebug(5006) << k_funcinfo << "Reading index for " << label() << endl;
00221   }
00222   Q_INT32 len;
00223   KMMsgInfo* mi;
00224 
00225   assert(mIndexStream != 0);
00226   rewind(mIndexStream);
00227 
00228   clearIndex();
00229   int version;
00230 
00231   setDirty( false );
00232 
00233   if (!readIndexHeader(&version)) return false;
00234   //kdDebug(5006) << "Index version for " << label() << " is " << version << endl;
00235 
00236   mUnreadMsgs = 0;
00237   mTotalMsgs = 0;
00238   mHeaderOffset = ftell(mIndexStream);
00239 
00240   clearIndex();
00241   while (!feof(mIndexStream))
00242   {
00243     mi = 0;
00244     if(version >= 1505) {
00245       if(!fread(&len, sizeof(len), 1, mIndexStream)) {
00246         // Seems to be normal?
00247         // kdDebug(5006) << k_funcinfo << " Unable to read length field!" << endl;
00248         break;
00249       }
00250 
00251       if (mIndexSwapByteOrder)
00252         len = kmail_swap_32(len);
00253 
00254       off_t offs = ftell(mIndexStream);
00255       if(fseek(mIndexStream, len, SEEK_CUR)) {
00256         kdDebug(5006) << k_funcinfo << " Unable to seek to the end of the message!" << endl;
00257         break;
00258       }
00259       mi = new KMMsgInfo(folder(), offs, len);
00260     }
00261     else
00262     {
00263       QCString line(MAX_LINE);
00264       fgets(line.data(), MAX_LINE, mIndexStream);
00265       if (feof(mIndexStream)) break;
00266       if (*line.data() == '\0') {
00267         fclose(mIndexStream);
00268         mIndexStream = 0;
00269         clearIndex();
00270         return false;
00271       }
00272       mi = new KMMsgInfo(folder());
00273       mi->compat_fromOldIndexString(line, mConvertToUtf8);
00274     }
00275     if(!mi) {
00276       kdDebug(5006) << k_funcinfo << " Unable to create message info object!" << endl;
00277       break;
00278     }
00279 
00280     if (mi->isDeleted())
00281     {
00282       delete mi;  // skip messages that are marked as deleted
00283       setDirty( true );
00284       needsCompact = true;  //We have deleted messages - needs to be compacted
00285       continue;
00286     }
00287 #ifdef OBSOLETE
00288     else if (mi->isNew())
00289     {
00290       mi->setStatus(KMMsgStatusUnread);
00291       mi->setDirty(false);
00292     }
00293 #endif
00294     if ((mi->isNew()) || (mi->isUnread()) ||
00295         (folder() == kmkernel->outboxFolder()))
00296     {
00297       ++mUnreadMsgs;
00298       if (mUnreadMsgs == 0) ++mUnreadMsgs;
00299     }
00300     mMsgList.append(mi, false);
00301   }
00302   if( version < 1505)
00303   {
00304     mConvertToUtf8 = false;
00305     setDirty( true );
00306     writeIndex();
00307   }
00308 
00309   if ( version < 1507 ) {
00310     updateInvitationAndAddressFieldsFromContents();
00311     setDirty( true );
00312     writeIndex();
00313   }
00314 
00315   mTotalMsgs = mMsgList.count();
00316   if ( contentsType() != KMail::ContentsTypeMail ) {
00317     kdDebug(5006) << k_funcinfo << "Done reading the index for " << label() << ", we have " << mTotalMsgs << " messages." << endl;
00318   }
00319   return true;
00320 }
00321 
00322 
00323 int KMFolderIndex::count(bool cache) const
00324 {
00325   int res = FolderStorage::count(cache);
00326   if (res == -1)
00327     res = mMsgList.count();
00328   return res;
00329 }
00330 
00331 
00332 bool KMFolderIndex::readIndexHeader(int *gv)
00333 {
00334   int indexVersion;
00335   assert(mIndexStream != 0);
00336   mIndexSwapByteOrder = false;
00337   mIndexSizeOfLong = sizeof(long);
00338 
00339   int ret = fscanf(mIndexStream, "# KMail-Index V%d\n", &indexVersion);
00340   if ( ret == EOF || ret == 0 )
00341       return false; // index file has invalid header
00342   if(gv)
00343       *gv = indexVersion;
00344 
00345   // Check if the index is corrupted ("not compactable") and recreate it if necessary. See
00346   // FolderStorage::getMsg() for the detection code.
00347   if ( !mCompactable ) {
00348     kdWarning(5006) << "Index file " << indexLocation() << " is corrupted!!. Re-creating it." << endl;
00349     recreateIndex( false /* don't call readIndex() afterwards */ );
00350     return false;
00351   }
00352 
00353   if (indexVersion < 1505 ) {
00354       if(indexVersion == 1503) {
00355       kdDebug(5006) << "Converting old index file " << indexLocation() << " to utf-8" << endl;
00356       mConvertToUtf8 = true;
00357       }
00358       return true;
00359   } else if (indexVersion == 1505) {
00360   } else if (indexVersion < INDEX_VERSION && indexVersion != 1506) {
00361       kdDebug(5006) << "Index file " << indexLocation() << " is out of date. Re-creating it." << endl;
00362       createIndexFromContents();
00363       return false;
00364   } else if(indexVersion > INDEX_VERSION) {
00365       kapp->setOverrideCursor(KCursor::arrowCursor());
00366       int r = KMessageBox::questionYesNo(0,
00367                      i18n(
00368                         "The mail index for '%1' is from an unknown version of KMail (%2).\n"
00369                         "This index can be regenerated from your mail folder, but some "
00370                         "information, including status flags, may be lost. Do you wish "
00371                         "to downgrade your index file?") .arg(name()) .arg(indexVersion), QString::null, i18n("Downgrade"), i18n("Do Not Downgrade") );
00372       kapp->restoreOverrideCursor();
00373       if (r == KMessageBox::Yes)
00374       createIndexFromContents();
00375       return false;
00376   }
00377   else {
00378       // Header
00379       Q_UINT32 byteOrder = 0;
00380       Q_UINT32 sizeOfLong = sizeof(long); // default
00381 
00382       Q_UINT32 header_length = 0;
00383       fseek(mIndexStream, sizeof(char), SEEK_CUR );
00384       fread(&header_length, sizeof(header_length), 1, mIndexStream);
00385       if (header_length > 0xFFFF)
00386          header_length = kmail_swap_32(header_length);
00387 
00388       off_t endOfHeader = ftell(mIndexStream) + header_length;
00389 
00390       bool needs_update = true;
00391       // Process available header parts
00392       if (header_length >= sizeof(byteOrder))
00393       {
00394          fread(&byteOrder, sizeof(byteOrder), 1, mIndexStream);
00395          mIndexSwapByteOrder = (byteOrder == 0x78563412);
00396          header_length -= sizeof(byteOrder);
00397 
00398          if (header_length >= sizeof(sizeOfLong))
00399          {
00400             fread(&sizeOfLong, sizeof(sizeOfLong), 1, mIndexStream);
00401             if (mIndexSwapByteOrder)
00402                sizeOfLong = kmail_swap_32(sizeOfLong);
00403             mIndexSizeOfLong = sizeOfLong;
00404             header_length -= sizeof(sizeOfLong);
00405             needs_update = false;
00406          }
00407       }
00408       if (needs_update || mIndexSwapByteOrder || (mIndexSizeOfLong != sizeof(long)))
00409     setDirty( true );
00410       // Seek to end of header
00411       fseek(mIndexStream, endOfHeader, SEEK_SET );
00412 
00413       if (mIndexSwapByteOrder)
00414          kdDebug(5006) << "Index File has byte order swapped!" << endl;
00415       if (mIndexSizeOfLong != sizeof(long))
00416          kdDebug(5006) << "Index File sizeOfLong is " << mIndexSizeOfLong << " while sizeof(long) is " << sizeof(long) << " !" << endl;
00417 
00418   }
00419   return true;
00420 }
00421 
00422 
00423 #ifdef HAVE_MMAP
00424 bool KMFolderIndex::updateIndexStreamPtr(bool just_close)
00425 #else
00426 bool KMFolderIndex::updateIndexStreamPtr(bool)
00427 #endif
00428 {
00429     // We touch the folder, otherwise the index is regenerated, if KMail is
00430     // running, while the clock switches from daylight savings time to normal time
00431     utime(QFile::encodeName(location()), 0);
00432     utime(QFile::encodeName(indexLocation()), 0);
00433     utime(QFile::encodeName( KMMsgDict::getFolderIdsLocation( *this ) ), 0);
00434 
00435   mIndexSwapByteOrder = false;
00436 #ifdef HAVE_MMAP
00437     if(just_close) {
00438     if(mIndexStreamPtr)
00439         munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength);
00440     mIndexStreamPtr = 0;
00441     mIndexStreamPtrLength = 0;
00442     return true;
00443     }
00444 
00445     assert(mIndexStream);
00446     struct stat stat_buf;
00447     if(fstat(fileno(mIndexStream), &stat_buf) == -1) {
00448     if(mIndexStreamPtr)
00449         munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength);
00450     mIndexStreamPtr = 0;
00451     mIndexStreamPtrLength = 0;
00452     return false;
00453     }
00454     if(mIndexStreamPtr)
00455     munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength);
00456     mIndexStreamPtrLength = stat_buf.st_size;
00457     mIndexStreamPtr = (uchar *)mmap(0, mIndexStreamPtrLength, PROT_READ, MAP_SHARED,
00458                     fileno(mIndexStream), 0);
00459     if(mIndexStreamPtr == MAP_FAILED) {
00460     mIndexStreamPtr = 0;
00461     mIndexStreamPtrLength = 0;
00462     return false;
00463     }
00464 #endif
00465     return true;
00466 }
00467 
00468 
00469 KMFolderIndex::IndexStatus KMFolderIndex::indexStatus()
00470 {
00471     QFileInfo contInfo(location());
00472     QFileInfo indInfo(indexLocation());
00473 
00474     if (!contInfo.exists()) return KMFolderIndex::IndexOk;
00475     if (!indInfo.exists()) return KMFolderIndex::IndexMissing;
00476 
00477     return ( contInfo.lastModified() > indInfo.lastModified() )
00478         ? KMFolderIndex::IndexTooOld
00479         : KMFolderIndex::IndexOk;
00480 }
00481 
00482 void KMFolderIndex::clearIndex(bool autoDelete, bool syncDict)
00483 {
00484     mMsgList.clear(autoDelete, syncDict);
00485 }
00486 
00487 
00488 void KMFolderIndex::truncateIndex()
00489 {
00490   if ( mHeaderOffset )
00491     truncate(QFile::encodeName(indexLocation()), mHeaderOffset);
00492   else
00493     // The index file wasn't opened, so we don't know the header offset.
00494     // So let's just create a new empty index.
00495     writeIndex( true );
00496 }
00497 
00498 void KMFolderIndex::fillMessageDict()
00499 {
00500   open("fillDict");
00501   for (unsigned int idx = 0; idx < mMsgList.high(); idx++)
00502     if ( mMsgList.at( idx ) )
00503       KMMsgDict::mutableInstance()->insert(0, mMsgList.at( idx ), idx);
00504   close("fillDict");
00505 }
00506 
00507 
00508 KMMsgInfo* KMFolderIndex::setIndexEntry( int idx, KMMessage *msg )
00509 {
00510   KMMsgInfo *msgInfo = msg->msgInfo();
00511   if ( !msgInfo )
00512     msgInfo = new KMMsgInfo( folder() );
00513 
00514   *msgInfo = *msg;
00515   mMsgList.set( idx, msgInfo );
00516   msg->setMsgInfo( 0 );
00517   delete msg;
00518   return msgInfo;
00519 }
00520 
00521 void KMFolderIndex::recreateIndex( bool readIndexAfterwards )
00522 {
00523   kapp->setOverrideCursor(KCursor::arrowCursor());
00524   KMessageBox::error(0,
00525        i18n("The mail index for '%1' is corrupted and will be regenerated now, "
00526             "but some information, including status flags, will be lost.").arg(name()));
00527   kapp->restoreOverrideCursor();
00528   createIndexFromContents();
00529   if ( readIndexAfterwards ) {
00530     readIndex();
00531   }
00532 
00533   // Clear the corrupted flag
00534   mCompactable = true;
00535   writeConfig();
00536 }
00537 
00538 void KMFolderIndex::updateInvitationAndAddressFieldsFromContents()
00539 {
00540   kdDebug(5006) << "Updating index for " << label() << ", this might take a while." << endl;
00541   for ( uint i = 0; i < mMsgList.size(); i++ ) {
00542     KMMsgInfo * const msgInfo = dynamic_cast<KMMsgInfo*>( mMsgList[i] );
00543     if ( msgInfo ) {
00544       DwString msgString( getDwString( i ) );
00545       if ( msgString.size() > 0 ) {
00546         KMMessage msg;
00547         msg.fromDwString( msgString, false );
00548         msg.updateInvitationState();
00549         if ( msg.status() & KMMsgStatusHasInvitation ) {
00550           msgInfo->setStatus( msgInfo->status() | KMMsgStatusHasInvitation );
00551         }
00552         if ( msg.status() & KMMsgStatusHasNoInvitation ) {
00553           msgInfo->setStatus( msgInfo->status() | KMMsgStatusHasNoInvitation );
00554         }
00555         msgInfo->setFrom( msg.from() );
00556         msgInfo->setTo( msg.to() );
00557       }
00558     }
00559   }
00560 }
00561 
00562 #include "kmfolderindex.moc"
KDE Home | KDE Accessibility Home | Description of Access Keys