// 7zUpdate.cpp
#include "StdAfx.h"
#include "../../../../C/CpuArch.h"
#include "../../../Common/Wildcard.h"
#include "../../Common/CreateCoder.h"
#include "../../Common/LimitedStreams.h"
#include "../../Common/ProgressUtils.h"
#include "../../Compress/CopyCoder.h"
#include "../Common/ItemNameUtils.h"
#include "../Common/OutStreamWithCRC.h"
#include "7zDecode.h"
#include "7zEncode.h"
#include "7zFolderInStream.h"
#include "7zHandler.h"
#include "7zOut.h"
#include "7zUpdate.h"
namespace NArchive {
namespace N7z {
#ifdef MY_CPU_X86_OR_AMD64
#define USE_86_FILTER
#endif
static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
UInt64 position, UInt64 size, ICompressProgressInfo *progress)
{
RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0));
CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream;
CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec);
streamSpec->SetStream(inStream);
streamSpec->Init(size);
NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder;
CMyComPtr<ICompressCoder> copyCoder = copyCoderSpec;
RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress));
return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL);
}
static int GetReverseSlashPos(const UString &name)
{
int slashPos = name.ReverseFind(L'/');
#ifdef _WIN32
int slash1Pos = name.ReverseFind(L'\\');
slashPos = MyMax(slashPos, slash1Pos);
#endif
return slashPos;
}
int CUpdateItem::GetExtensionPos() const
{
int slashPos = GetReverseSlashPos(Name);
int dotPos = Name.ReverseFind(L'.');
if (dotPos < 0 || (dotPos < slashPos && slashPos >= 0))
return Name.Len();
return dotPos + 1;
}
UString CUpdateItem::GetExtension() const
{
return Name.Ptr(GetExtensionPos());
}
#define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; }
#define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
/*
static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
{
size_t c1 = a1.GetCapacity();
size_t c2 = a2.GetCapacity();
RINOZ_COMP(c1, c2);
for (size_t i = 0; i < c1; i++)
RINOZ_COMP(a1[i], a2[i]);
return 0;
}
static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
{
RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
RINOZ_COMP(c1.MethodID, c2.MethodID);
return CompareBuffers(c1.Props, c2.Props);
}
static int CompareBindPairs(const CBindPair &b1, const CBindPair &b2)
{
RINOZ_COMP(b1.InIndex, b2.InIndex);
return MyCompare(b1.OutIndex, b2.OutIndex);
}
static int CompareFolders(const CFolder &f1, const CFolder &f2)
{
int s1 = f1.Coders.Size();
int s2 = f2.Coders.Size();
RINOZ_COMP(s1, s2);
int i;
for (i = 0; i < s1; i++)
RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
s1 = f1.BindPairs.Size();
s2 = f2.BindPairs.Size();
RINOZ_COMP(s1, s2);
for (i = 0; i < s1; i++)
RINOZ(CompareBindPairs(f1.BindPairs[i], f2.BindPairs[i]));
return 0;
}
*/
/*
static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
{
return CompareFileNames(f1.Name, f2.Name);
}
*/
struct CFolderRepack
{
int FolderIndex;
int Group;
CNum NumCopyFiles;
};
static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void * /* param */)
{
RINOZ_COMP(p1->Group, p2->Group);
int i1 = p1->FolderIndex;
int i2 = p2->FolderIndex;
/*
// In that version we don't want to parse folders here, so we don't compare folders
// probably it must be improved in future
const CDbEx &db = *(const CDbEx *)param;
RINOZ(CompareFolders(
db.Folders[i1],
db.Folders[i2]));
*/
return MyCompare(i1, i2);
/*
RINOZ_COMP(
db.NumUnpackStreamsVector[i1],
db.NumUnpackStreamsVector[i2]);
if (db.NumUnpackStreamsVector[i1] == 0)
return 0;
return CompareFiles(
db.Files[db.FolderStartFileIndex[i1]],
db.Files[db.FolderStartFileIndex[i2]]);
*/
}
/*
we sort empty files and dirs in such order:
- Dir.NonAnti (name sorted)
- File.NonAnti (name sorted)
- File.Anti (name sorted)
- Dir.Anti (reverse name sorted)
*/
static int CompareEmptyItems(const int *p1, const int *p2, void *param)
{
const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
const CUpdateItem &u1 = updateItems[*p1];
const CUpdateItem &u2 = updateItems[*p2];
// NonAnti < Anti
if (u1.IsAnti != u2.IsAnti)
return (u1.IsAnti ? 1 : -1);
if (u1.IsDir != u2.IsDir)
{
// Dir.NonAnti < File < Dir.Anti
if (u1.IsDir)
return (u1.IsAnti ? 1 : -1);
return (u2.IsAnti ? -1 : 1);
}
int n = CompareFileNames(u1.Name, u2.Name);
return (u1.IsDir && u1.IsAnti) ? -n : n;
}
static const char *g_Exts =
" lzma 7z ace arc arj bz bz2 deb lzo lzx gz pak rpm sit tgz tbz tbz2 tgz cab ha lha lzh rar zoo"
" zip jar ear war msi"
" 3gp avi mov mpeg mpg mpe wmv"
" aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
" swf "
" chm hxi hxs"
" gif jpeg jpg jp2 png tiff bmp ico psd psp"
" awg ps eps cgm dxf svg vrml wmf emf ai md"
" cad dwg pps key sxi"
" max 3ds"
" iso bin nrg mdf img pdi tar cpio xpi"
" vfd vhd vud vmc vsv"
" vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
" inl inc idl acf asa h hpp hxx c cpp cxx rc java cs pas bas vb cls ctl frm dlg def"
" f77 f f90 f95"
" asm sql manifest dep "
" mak clw csproj vcproj sln dsp dsw "
" class "
" bat cmd"
" xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
" awk sed hta js php php3 php4 php5 phptml pl pm py pyo rb sh tcl vbs"
" text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
" sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
" abw afp cwk lwp wpd wps wpt wrf wri"
" abf afm bdf fon mgf otf pcf pfa snf ttf"
" dbf mdb nsf ntf wdb db fdb gdb"
" exe dll ocx vbx sfx sys tlb awx com obj lib out o so "
" pdb pch idb ncb opt";
static int GetExtIndex(const char *ext)
{
int extIndex = 1;
const char *p = g_Exts;
for (;;)
{
char c = *p++;
if (c == 0)
return extIndex;
if (c == ' ')
continue;
int pos = 0;
for (;;)
{
char c2 = ext[pos++];
if (c2 == 0 && (c == 0 || c == ' '))
return extIndex;
if (c != c2)
break;
c = *p++;
}
extIndex++;
for (;;)
{
if (c == 0)
return extIndex;
if (c == ' ')
break;
c = *p++;
}
}
}
struct CRefItem
{
const CUpdateItem *UpdateItem;
UInt32 Index;
UInt32 ExtensionPos;
UInt32 NamePos;
unsigned ExtensionIndex;
CRefItem() {};
CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
UpdateItem(&ui),
Index(index),
ExtensionPos(0),
NamePos(0),
ExtensionIndex(0)
{
if (sortByType)
{
int slashPos = GetReverseSlashPos(ui.Name);
NamePos = slashPos + 1;
int dotPos = ui.Name.ReverseFind(L'.');
if (dotPos < 0 || dotPos < slashPos)
ExtensionPos = ui.Name.Len();
else
{
ExtensionPos = dotPos + 1;
if (ExtensionPos != ui.Name.Len())
{
AString s;
for (unsigned pos = ExtensionPos;; pos++)
{
wchar_t c = ui.Name[pos];
if (c >= 0x80)
break;
if (c == 0)
{
ExtensionIndex = GetExtIndex(s);
break;
}
s += (char)MyCharLower_Ascii((char)c);
}
}
}
}
}
};
struct CSortParam
{
// const CObjectVector<CTreeFolder> *TreeFolders;
bool SortByType;
};
/*
we sort files in such order:
- Dir.NonAnti (name sorted)
- alt streams
- Dirs
- Dir.Anti (reverse name sorted)
*/
static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
{
const CRefItem &a1 = *p1;
const CRefItem &a2 = *p2;
const CUpdateItem &u1 = *a1.UpdateItem;
const CUpdateItem &u2 = *a2.UpdateItem;
/*
if (u1.IsAltStream != u2.IsAltStream)
return u1.IsAltStream ? 1 : -1;
*/
// Actually there are no dirs that time. They were stored in other steps
// So that code is unused?
if (u1.IsDir != u2.IsDir)
return u1.IsDir ? 1 : -1;
if (u1.IsDir)
{
if (u1.IsAnti != u2.IsAnti)
return (u1.IsAnti ? 1 : -1);
int n = CompareFileNames(u1.Name, u2.Name);
return -n;
}
// bool sortByType = *(bool *)param;
const CSortParam *sortParam = (const CSortParam *)param;
bool sortByType = sortParam->SortByType;
if (sortByType)
{
RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex);
RINOZ(CompareFileNames(u1.Name.Ptr(a1.ExtensionPos), u2.Name.Ptr(a2.ExtensionPos)));
RINOZ(CompareFileNames(u1.Name.Ptr(a1.NamePos), u2.Name.Ptr(a2.NamePos)));
if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime);
RINOZ_COMP(u1.Size, u2.Size);
}
/*
int par1 = a1.UpdateItem->ParentFolderIndex;
int par2 = a2.UpdateItem->ParentFolderIndex;
const CTreeFolder &tf1 = (*sortParam->TreeFolders)[par1];
const CTreeFolder &tf2 = (*sortParam->TreeFolders)[par2];
int b1 = tf1.SortIndex, e1 = tf1.SortIndexEnd;
int b2 = tf2.SortIndex, e2 = tf2.SortIndexEnd;
if (b1 < b2)
{
if (e1 <= b2)
return -1;
// p2 in p1
int par = par2;
for (;;)
{
const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
par = tf.Parent;
if (par == par1)
{
RINOZ(CompareFileNames(u1.Name, tf.Name));
break;
}
}
}
else if (b2 < b1)
{
if (e2 <= b1)
return 1;
// p1 in p2
int par = par1;
for (;;)
{
const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
par = tf.Parent;
if (par == par2)
{
RINOZ(CompareFileNames(tf.Name, u2.Name));
break;
}
}
}
*/
// RINOZ_COMP(a1.UpdateItem->ParentSortIndex, a2.UpdateItem->ParentSortIndex);
RINOK(CompareFileNames(u1.Name, u2.Name));
RINOZ_COMP(a1.UpdateItem->IndexInClient, a2.UpdateItem->IndexInClient);
RINOZ_COMP(a1.UpdateItem->IndexInArchive, a2.UpdateItem->IndexInArchive);
return 0;
}
struct CSolidGroup
{
CRecordVector<UInt32> Indices;
};
static const wchar_t *g_ExeExts[] =
{
L"dll"
, L"exe"
, L"ocx"
, L"sfx"
, L"sys"
};
static bool IsExeExt(const wchar_t *ext)
{
for (int i = 0; i < ARRAY_SIZE(g_ExeExts); i++)
if (MyStringCompareNoCase(ext, g_ExeExts[i]) == 0)
return true;
return false;
}
static inline void GetMethodFull(UInt64 methodID, UInt32 numInStreams, CMethodFull &m)
{
m.Id = methodID;
m.NumInStreams = numInStreams;
m.NumOutStreams = 1;
}
static void AddBcj2Methods(CCompressionMethodMode &mode)
{
CMethodFull m;
GetMethodFull(k_LZMA, 1, m);
m.AddProp32(NCoderPropID::kDictionarySize, 1 << 20);
m.AddProp32(NCoderPropID::kNumFastBytes, 128);
m.AddProp32(NCoderPropID::kNumThreads, 1);
m.AddProp32(NCoderPropID::kLitPosBits, 2);
m.AddProp32(NCoderPropID::kLitContextBits, 0);
// m.AddPropString(NCoderPropID::kMatchFinder, L"BT2");
mode.Methods.Add(m);
mode.Methods.Add(m);
CBind bind;
bind.OutCoder = 0;
bind.InStream = 0;
bind.InCoder = 1; bind.OutStream = 0; mode.Binds.Add(bind);
bind.InCoder = 2; bind.OutStream = 1; mode.Binds.Add(bind);
bind.InCoder = 3; bind.OutStream = 2; mode.Binds.Add(bind);
}
static void MakeExeMethod(CCompressionMethodMode &mode,
bool useFilters, bool addFilter, bool bcj2Filter)
{
if (!mode.Binds.IsEmpty() || !useFilters || mode.Methods.Size() > 2)
return;
if (mode.Methods.Size() == 2)
{
if (mode.Methods[0].Id == k_BCJ2)
AddBcj2Methods(mode);
return;
}
if (!addFilter)
return;
bcj2Filter = bcj2Filter;
#ifdef USE_86_FILTER
if (bcj2Filter)
{
CMethodFull m;
GetMethodFull(k_BCJ2, 4, m);
mode.Methods.Insert(0, m);
AddBcj2Methods(mode);
}
else
{
CMethodFull m;
GetMethodFull(k_BCJ, 1, m);
mode.Methods.Insert(0, m);
CBind bind;
bind.OutCoder = 0;
bind.InStream = 0;
bind.InCoder = 1;
bind.OutStream = 0;
mode.Binds.Add(bind);
}
#endif
}
static void FromUpdateItemToFileItem(const CUpdateItem &ui,
CFileItem &file, CFileItem2 &file2)
{
if (ui.AttribDefined)
file.SetAttrib(ui.Attrib);
file2.CTime = ui.CTime; file2.CTimeDefined = ui.CTimeDefined;
file2.ATime = ui.ATime; file2.ATimeDefined = ui.ATimeDefined;
file2.MTime = ui.MTime; file2.MTimeDefined = ui.MTimeDefined;
file2.IsAnti = ui.IsAnti;
// file2.IsAux = false;
file2.StartPosDefined = false;
file.Size = ui.Size;
file.IsDir = ui.IsDir;
file.HasStream = ui.HasStream();
// file.IsAltStream = ui.IsAltStream;
}
class CFolderOutStream2:
public ISequentialOutStream,
public CMyUnknownImp
{
COutStreamWithCRC *_crcStreamSpec;
CMyComPtr<ISequentialOutStream> _crcStream;
const CDbEx *_db;
const CBoolVector *_extractStatuses;
CMyComPtr<ISequentialOutStream> _outStream;
UInt32 _startIndex;
unsigned _currentIndex;
bool _fileIsOpen;
UInt64 _rem;
void OpenFile();
void CloseFile();
HRESULT CloseFileAndSetResult();
HRESULT ProcessEmptyFiles();
public:
MY_UNKNOWN_IMP
CFolderOutStream2()
{
_crcStreamSpec = new COutStreamWithCRC;
_crcStream = _crcStreamSpec;
}
HRESULT Init(const CDbEx *db, UInt32 startIndex,
const CBoolVector *extractStatuses, ISequentialOutStream *outStream);
void ReleaseOutStream();
HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
};
HRESULT CFolderOutStream2::Init(const CDbEx *db, UInt32 startIndex,
const CBoolVector *extractStatuses, ISequentialOutStream *outStream)
{
_db = db;
_startIndex = startIndex;
_extractStatuses = extractStatuses;
_outStream = outStream;
_currentIndex = 0;
_fileIsOpen = false;
return ProcessEmptyFiles();
}
void CFolderOutStream2::ReleaseOutStream()
{
_outStream.Release();
_crcStreamSpec->ReleaseStream();
}
void CFolderOutStream2::OpenFile()
{
_crcStreamSpec->SetStream((*_extractStatuses)[_currentIndex] ? _outStream : NULL);
_crcStreamSpec->Init(true);
_fileIsOpen = true;
_rem = _db->Files[_startIndex + _currentIndex].Size;
}
void CFolderOutStream2::CloseFile()
{
_crcStreamSpec->ReleaseStream();
_fileIsOpen = false;
_currentIndex++;
}
HRESULT CFolderOutStream2::CloseFileAndSetResult()
{
const CFileItem &file = _db->Files[_startIndex + _currentIndex];
CloseFile();
return (file.IsDir || !file.CrcDefined || file.Crc == _crcStreamSpec->GetCRC()) ? S_OK: S_FALSE;
}
HRESULT CFolderOutStream2::ProcessEmptyFiles()
{
while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
{
OpenFile();
RINOK(CloseFileAndSetResult());
}
return S_OK;
}
STDMETHODIMP CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize)
{
if (processedSize != NULL)
*processedSize = 0;
while (size != 0)
{
if (_fileIsOpen)
{
UInt32 cur = size < _rem ? size : (UInt32)_rem;
RINOK(_crcStream->Write(data, cur, &cur));
if (cur == 0)
break;
data = (const Byte *)data + cur;
size -= cur;
_rem -= cur;
if (processedSize != NULL)
*processedSize += cur;
if (_rem == 0)
{
RINOK(CloseFileAndSetResult());
RINOK(ProcessEmptyFiles());
continue;
}
}
else
{
RINOK(ProcessEmptyFiles());
if (_currentIndex == _extractStatuses->Size())
{
// we don't support partial extracting
return E_FAIL;
}
OpenFile();
}
}
return S_OK;
}
class CThreadDecoder: public CVirtThread
{
public:
HRESULT Result;
CMyComPtr<IInStream> InStream;
CFolderOutStream2 *FosSpec;
CMyComPtr<ISequentialOutStream> Fos;
UInt64 StartPos;
const CFolders *Folders;
int FolderIndex;
#ifndef _NO_CRYPTO
CMyComPtr<ICryptoGetTextPassword> getTextPassword;
#endif
DECL_EXTERNAL_CODECS_LOC_VARS2;
CDecoder Decoder;
#ifndef _7ZIP_ST
bool MtMode;
UInt32 NumThreads;
#endif
CThreadDecoder():
Decoder(true)
{
#ifndef _7ZIP_ST
MtMode = false;
NumThreads = 1;
#endif
FosSpec = new CFolderOutStream2;
Fos = FosSpec;
Result = E_FAIL;
}
~CThreadDecoder() { CVirtThread::WaitThreadFinish(); }
virtual void Execute();
};
void CThreadDecoder::Execute()
{
try
{
#ifndef _NO_CRYPTO
bool isEncrypted = false;
bool passwordIsDefined = false;
#endif
Result = Decoder.Decode(
EXTERNAL_CODECS_LOC_VARS
InStream,
StartPos,
*Folders, FolderIndex,
Fos,
NULL
_7Z_DECODER_CRYPRO_VARS
#ifndef _7ZIP_ST
, MtMode, NumThreads
#endif
);
}
catch(...)
{
Result = E_FAIL;
}
if (Result == S_OK)
Result = FosSpec->CheckFinishedState();
FosSpec->ReleaseOutStream();
}
bool static Is86FilteredFolder(const CFolder &f)
{
FOR_VECTOR(i, f.Coders)
{
CMethodId m = f.Coders[i].MethodID;
if (m == k_BCJ || m == k_BCJ2)
return true;
}
return false;
}
#ifndef _NO_CRYPTO
class CCryptoGetTextPassword:
public ICryptoGetTextPassword,
public CMyUnknownImp
{
public:
UString Password;
MY_UNKNOWN_IMP
STDMETHOD(CryptoGetTextPassword)(BSTR *password);
};
STDMETHODIMP CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password)
{
return StringToBstr(Password, password);
}
#endif
static const int kNumGroupsMax = 4;
static bool Is86Group(int group) { return (group & 1) != 0; }
static bool IsEncryptedGroup(int group) { return (group & 2) != 0; }
static int GetGroupIndex(bool encrypted, int bcjFiltered)
{ return (encrypted ? 2 : 0) + (bcjFiltered ? 1 : 0); }
static void GetFile(const CDatabase &inDb, int index, CFileItem &file, CFileItem2 &file2)
{
file = inDb.Files[index];
file2.CTimeDefined = inDb.CTime.GetItem(index, file2.CTime);
file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime);
file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime);
file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos);
file2.IsAnti = inDb.IsItemAnti(index);
// file2.IsAux = inDb.IsItemAux(index);
}
HRESULT Update(
DECL_EXTERNAL_CODECS_LOC_VARS
IInStream *inStream,
const CDbEx *db,
const CObjectVector<CUpdateItem> &updateItems,
// const CObjectVector<CTreeFolder> &treeFolders,
// const CUniqBlocks &secureBlocks,
COutArchive &archive,
CArchiveDatabaseOut &newDatabase,
ISequentialOutStream *seqOutStream,
IArchiveUpdateCallback *updateCallback,
const CUpdateOptions &options
#ifndef _NO_CRYPTO
, ICryptoGetTextPassword *getDecoderPassword
#endif
)
{
UInt64 numSolidFiles = options.NumSolidFiles;
if (numSolidFiles == 0)
numSolidFiles = 1;
// size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes();
/*
CMyComPtr<IOutStream> outStream;
RINOK(seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStream));
if (!outStream)
return E_NOTIMPL;
*/
UInt64 startBlockSize = db != 0 ? db->ArcInfo.StartPosition: 0;
if (startBlockSize > 0 && !options.RemoveSfxBlock)
{
RINOK(WriteRange(inStream, seqOutStream, 0, startBlockSize, NULL));
}
CIntArr fileIndexToUpdateIndexMap;
CRecordVector<CFolderRepack> folderRefs;
UInt64 complexity = 0;
UInt64 inSizeForReduce2 = 0;
bool needEncryptedRepack = false;
if (db != 0)
{
fileIndexToUpdateIndexMap.Alloc(db->Files.Size());
unsigned i;
for (i = 0; i < db->Files.Size(); i++)
fileIndexToUpdateIndexMap[i] = -1;
for (i = 0; i < updateItems.Size(); i++)
{
int index = updateItems[i].IndexInArchive;
if (index != -1)
fileIndexToUpdateIndexMap[index] = i;
}
for (i = 0; i < (int)db->NumFolders; i++)
{
CNum indexInFolder = 0;
CNum numCopyItems = 0;
CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
UInt64 repackSize = 0;
for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
{
const CFileItem &file = db->Files[fi];
if (file.HasStream)
{
indexInFolder++;
int updateIndex = fileIndexToUpdateIndexMap[fi];
if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
{
numCopyItems++;
repackSize += file.Size;
}
}
}
if (numCopyItems == 0)
continue;
CFolderRepack rep;
rep.FolderIndex = i;
rep.NumCopyFiles = numCopyItems;
CFolder f;
db->ParseFolderInfo(i, f);
bool isEncrypted = f.IsEncrypted();
rep.Group = GetGroupIndex(isEncrypted, Is86FilteredFolder(f));
folderRefs.Add(rep);
if (numCopyItems == numUnpackStreams)
complexity += db->GetFolderFullPackSize(i);
else
{
complexity += repackSize;
if (repackSize > inSizeForReduce2)
inSizeForReduce2 = repackSize;
if (isEncrypted)
needEncryptedRepack = true;
}
}
folderRefs.Sort(CompareFolderRepacks, (void *)db);
}
UInt64 inSizeForReduce = 0;
unsigned i;
for (i = 0; i < updateItems.Size(); i++)
{
const CUpdateItem &ui = updateItems[i];
if (ui.NewData)
{
complexity += ui.Size;
if (numSolidFiles != 1)
inSizeForReduce += ui.Size;
else if (ui.Size > inSizeForReduce)
inSizeForReduce = ui.Size;
}
}
if (inSizeForReduce2 > inSizeForReduce)
inSizeForReduce = inSizeForReduce2;
RINOK(updateCallback->SetTotal(complexity));
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(updateCallback, true);
CStreamBinder sb;
RINOK(sb.CreateEvents());
CThreadDecoder threadDecoder;
if (!folderRefs.IsEmpty())
{
#ifdef EXTERNAL_CODECS
threadDecoder.__externalCodecs = __externalCodecs;
#endif
RINOK(threadDecoder.Create());
}
CObjectVector<CSolidGroup> groups;
for (i = 0; i < kNumGroupsMax; i++)
groups.AddNew();
{
// ---------- Split files to groups ----------
bool useFilters = options.UseFilters;
const CCompressionMethodMode &method = *options.Method;
if (method.Methods.Size() != 1 || method.Binds.Size() != 0)
useFilters = false;
for (i = 0; i < updateItems.Size(); i++)
{
const CUpdateItem &ui = updateItems[i];
if (!ui.NewData || !ui.HasStream())
continue;
bool filteredGroup = false;
if (useFilters)
{
int dotPos = ui.Name.ReverseFind(L'.');
if (dotPos >= 0)
filteredGroup = IsExeExt(ui.Name.Ptr(dotPos + 1));
}
groups[GetGroupIndex(method.PasswordIsDefined, filteredGroup)].Indices.Add(i);
}
}
#ifndef _NO_CRYPTO
CCryptoGetTextPassword *getPasswordSpec = NULL;
if (needEncryptedRepack)
{
getPasswordSpec = new CCryptoGetTextPassword;
threadDecoder.getTextPassword = getPasswordSpec;
if (options.Method->PasswordIsDefined)
getPasswordSpec->Password = options.Method->Password;
else
{
if (!getDecoderPassword)
return E_NOTIMPL;
CMyComBSTR password;
RINOK(getDecoderPassword->CryptoGetTextPassword(&password));
if ((BSTR)password)
getPasswordSpec->Password = password;
}
}
#endif
// ---------- Compress ----------
RINOK(archive.Create(seqOutStream, false));
RINOK(archive.SkipPrefixArchiveHeader());
/*
CIntVector treeFolderToArcIndex;
treeFolderToArcIndex.Reserve(treeFolders.Size());
for (i = 0; i < treeFolders.Size(); i++)
treeFolderToArcIndex.Add(-1);
// ---------- Write Tree (only AUX dirs) ----------
for (i = 1; i < treeFolders.Size(); i++)
{
const CTreeFolder &treeFolder = treeFolders[i];
CFileItem file;
CFileItem2 file2;
file2.Init();
int secureID = 0;
if (treeFolder.UpdateItemIndex < 0)
{
// we can store virtual dir item wuthout attrib, but we want all items have attrib.
file.SetAttrib(FILE_ATTRIBUTE_DIRECTORY);
file2.IsAux = true;
}
else
{
const CUpdateItem &ui = updateItems[treeFolder.UpdateItemIndex];
// if item is not dir, then it's parent for alt streams.
// we will write such items later
if (!ui.IsDir)
continue;
secureID = ui.SecureIndex;
if (ui.NewProps)
FromUpdateItemToFileItem(ui, file, file2);
else
GetFile(*db, ui.IndexInArchive, file, file2);
}
file.Size = 0;
file.HasStream = false;
file.IsDir = true;
file.Parent = treeFolder.Parent;
treeFolderToArcIndex[i] = newDatabase.Files.Size();
newDatabase.AddFile(file, file2, treeFolder.Name);
if (totalSecureDataSize != 0)
newDatabase.SecureIDs.Add(secureID);
}
*/
{
/* ---------- Write non-AUX dirs and Empty files ---------- */
CRecordVector<int> emptyRefs;
for (i = 0; i < updateItems.Size(); i++)
{
const CUpdateItem &ui = updateItems[i];
if (ui.NewData)
{
if (ui.HasStream())
continue;
}
else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream)
continue;
/*
if (ui.TreeFolderIndex >= 0)
continue;
*/
emptyRefs.Add(i);
}
emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
for (i = 0; i < emptyRefs.Size(); i++)
{
const CUpdateItem &ui = updateItems[emptyRefs[i]];
CFileItem file;
CFileItem2 file2;
UString name;
if (ui.NewProps)
{
FromUpdateItemToFileItem(ui, file, file2);
name = ui.Name;
}
else
{
GetFile(*db, ui.IndexInArchive, file, file2);
db->GetPath(ui.IndexInArchive, name);
}
/*
if (totalSecureDataSize != 0)
newDatabase.SecureIDs.Add(ui.SecureIndex);
file.Parent = ui.ParentFolderIndex;
*/
newDatabase.AddFile(file, file2, name);
}
}
unsigned folderRefIndex = 0;
lps->ProgressOffset = 0;
for (int groupIndex = 0; groupIndex < kNumGroupsMax; groupIndex++)
{
const CSolidGroup &group = groups[groupIndex];
CCompressionMethodMode method = *options.Method;
MakeExeMethod(method, options.UseFilters, Is86Group(groupIndex), options.MaxFilter);
if (IsEncryptedGroup(groupIndex))
{
if (!method.PasswordIsDefined)
{
#ifndef _NO_CRYPTO
if (getPasswordSpec)
method.Password = getPasswordSpec->Password;
#endif
method.PasswordIsDefined = true;
}
}
else
{
method.PasswordIsDefined = false;
method.Password.Empty();
}
CEncoder encoder(method);
for (; folderRefIndex < folderRefs.Size(); folderRefIndex++)
{
const CFolderRepack &rep = folderRefs[folderRefIndex];
if (rep.Group != groupIndex)
break;
int folderIndex = rep.FolderIndex;
if (rep.NumCopyFiles == db->NumUnpackStreamsVector[folderIndex])
{
UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
RINOK(WriteRange(inStream, archive.SeqStream,
db->GetFolderStreamPos(folderIndex, 0), packSize, progress));
lps->ProgressOffset += packSize;
CFolder &folder = newDatabase.Folders.AddNew();
db->ParseFolderInfo(folderIndex, folder);
CNum startIndex = db->FoStartPackStreamIndex[folderIndex];
for (unsigned j = 0; j < folder.PackStreams.Size(); j++)
{
newDatabase.PackSizes.Add(db->GetStreamPackSize(startIndex + j));
// newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
// newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
}
UInt32 indexStart = db->FoToCoderUnpackSizes[folderIndex];
UInt32 indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1];
for (; indexStart < indexEnd; indexStart++)
newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes[indexStart]);
}
else
{
CBoolVector extractStatuses;
CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
CNum indexInFolder = 0;
for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
{
bool needExtract = false;
if (db->Files[fi].HasStream)
{
indexInFolder++;
int updateIndex = fileIndexToUpdateIndexMap[fi];
if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
needExtract = true;
}
extractStatuses.Add(needExtract);
}
unsigned startPackIndex = newDatabase.PackSizes.Size();
UInt64 curUnpackSize;
{
CMyComPtr<ISequentialInStream> sbInStream;
{
CMyComPtr<ISequentialOutStream> sbOutStream;
sb.CreateStreams(&sbInStream, &sbOutStream);
sb.ReInit();
RINOK(threadDecoder.FosSpec->Init(db, db->FolderStartFileIndex[folderIndex], &extractStatuses, sbOutStream));
}
threadDecoder.InStream = inStream;
threadDecoder.Folders = (const CFolders *)db;
threadDecoder.FolderIndex = folderIndex;
threadDecoder.StartPos = db->ArcInfo.DataStartPosition; // db->GetFolderStreamPos(folderIndex, 0);
threadDecoder.Start();
RINOK(encoder.Encode(
EXTERNAL_CODECS_LOC_VARS
sbInStream, NULL, &inSizeForReduce,
newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curUnpackSize,
archive.SeqStream, newDatabase.PackSizes, progress));
threadDecoder.WaitExecuteFinish();
}
RINOK(threadDecoder.Result);
for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
lps->OutSize += newDatabase.PackSizes[startPackIndex];
lps->InSize += curUnpackSize;
}
newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
CNum indexInFolder = 0;
for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
{
CFileItem file;
CFileItem2 file2;
GetFile(*db, fi, file, file2);
UString name;
db->GetPath(fi, name);
if (file.HasStream)
{
indexInFolder++;
int updateIndex = fileIndexToUpdateIndexMap[fi];
if (updateIndex >= 0)
{
const CUpdateItem &ui = updateItems[updateIndex];
if (ui.NewData)
continue;
if (ui.NewProps)
{
CFileItem uf;
FromUpdateItemToFileItem(ui, uf, file2);
uf.Size = file.Size;
uf.Crc = file.Crc;
uf.CrcDefined = file.CrcDefined;
uf.HasStream = file.HasStream;
file = uf;
name = ui.Name;
}
/*
file.Parent = ui.ParentFolderIndex;
if (ui.TreeFolderIndex >= 0)
treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
if (totalSecureDataSize != 0)
newDatabase.SecureIDs.Add(ui.SecureIndex);
*/
newDatabase.AddFile(file, file2, name);
}
}
}
}
unsigned numFiles = group.Indices.Size();
if (numFiles == 0)
continue;
CRecordVector<CRefItem> refItems;
refItems.ClearAndSetSize(numFiles);
bool sortByType = (numSolidFiles > 1);
for (i = 0; i < numFiles; i++)
refItems[i] = CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType);
CSortParam sortParam;
// sortParam.TreeFolders = &treeFolders;
sortParam.SortByType = sortByType;
refItems.Sort(CompareUpdateItems, (void *)&sortParam);
CObjArray<UInt32> indices(numFiles);
for (i = 0; i < numFiles; i++)
{
UInt32 index = refItems[i].Index;
indices[i] = index;
/*
const CUpdateItem &ui = updateItems[index];
CFileItem file;
if (ui.NewProps)
FromUpdateItemToFileItem(ui, file);
else
file = db.Files[ui.IndexInArchive];
if (file.IsAnti || file.IsDir)
return E_FAIL;
newDatabase.Files.Add(file);
*/
}
for (i = 0; i < numFiles;)
{
UInt64 totalSize = 0;
int numSubFiles;
UString prevExtension;
for (numSubFiles = 0; i + numSubFiles < numFiles &&
numSubFiles < numSolidFiles; numSubFiles++)
{
const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
totalSize += ui.Size;
if (totalSize > options.NumSolidBytes)
break;
if (options.SolidExtension)
{
UString ext = ui.GetExtension();
if (numSubFiles == 0)
prevExtension = ext;
else
if (!ext.IsEqualToNoCase(prevExtension))
break;
}
}
if (numSubFiles < 1)
numSubFiles = 1;
CFolderInStream *inStreamSpec = new CFolderInStream;
CMyComPtr<ISequentialInStream> solidInStream(inStreamSpec);
inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
unsigned startPackIndex = newDatabase.PackSizes.Size();
UInt64 curFolderUnpackSize;
RINOK(encoder.Encode(
EXTERNAL_CODECS_LOC_VARS
solidInStream, NULL, &inSizeForReduce,
newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curFolderUnpackSize,
archive.SeqStream, newDatabase.PackSizes, progress));
for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
lps->OutSize += newDatabase.PackSizes[startPackIndex];
lps->InSize += curFolderUnpackSize;
// for ()
// newDatabase.PackCRCsDefined.Add(false);
// newDatabase.PackCRCs.Add(0);
CNum numUnpackStreams = 0;
for (int subIndex = 0; subIndex < numSubFiles; subIndex++)
{
const CUpdateItem &ui = updateItems[indices[i + subIndex]];
CFileItem file;
CFileItem2 file2;
UString name;
if (ui.NewProps)
{
FromUpdateItemToFileItem(ui, file, file2);
name = ui.Name;
}
else
{
GetFile(*db, ui.IndexInArchive, file, file2);
db->GetPath(ui.IndexInArchive, name);
}
if (file2.IsAnti || file.IsDir)
return E_FAIL;
/*
CFileItem &file = newDatabase.Files[
startFileIndexInDatabase + i + subIndex];
*/
if (!inStreamSpec->Processed[subIndex])
{
continue;
// file.Name += L".locked";
}
file.Crc = inStreamSpec->CRCs[subIndex];
file.Size = inStreamSpec->Sizes[subIndex];
if (file.Size != 0)
{
file.CrcDefined = true;
file.HasStream = true;
numUnpackStreams++;
}
else
{
file.CrcDefined = false;
file.HasStream = false;
}
/*
file.Parent = ui.ParentFolderIndex;
if (ui.TreeFolderIndex >= 0)
treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
if (totalSecureDataSize != 0)
newDatabase.SecureIDs.Add(ui.SecureIndex);
*/
newDatabase.AddFile(file, file2, name);
}
// numUnpackStreams = 0 is very bad case for locked files
// v3.13 doesn't understand it.
newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
i += numSubFiles;
}
}
if (folderRefIndex != folderRefs.Size())
return E_FAIL;
RINOK(lps->SetCur());
/*
folderRefs.ClearAndFree();
fileIndexToUpdateIndexMap.ClearAndFree();
groups.ClearAndFree();
*/
/*
for (i = 0; i < newDatabase.Files.Size(); i++)
{
CFileItem &file = newDatabase.Files[i];
file.Parent = treeFolderToArcIndex[file.Parent];
}
if (totalSecureDataSize != 0)
{
newDatabase.SecureBuf.SetCapacity(totalSecureDataSize);
size_t pos = 0;
newDatabase.SecureSizes.Reserve(secureBlocks.Sorted.Size());
for (i = 0; i < secureBlocks.Sorted.Size(); i++)
{
const CByteBuffer &buf = secureBlocks.Bufs[secureBlocks.Sorted[i]];
size_t size = buf.GetCapacity();
memcpy(newDatabase.SecureBuf + pos, buf, size);
newDatabase.SecureSizes.Add((UInt32)size);
pos += size;
}
}
*/
newDatabase.ReserveDown();
return S_OK;
}
}}