// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "net/http/http_cache.h" #include <algorithm> #include "base/compiler_specific.h" #if defined(OS_POSIX) #include <unistd.h> #endif #include "base/format_macros.h" #include "base/message_loop.h" #include "base/pickle.h" #include "base/ref_counted.h" #include "base/string_util.h" #include "net/base/io_buffer.h" #include "net/base/net_errors.h" #include "net/disk_cache/disk_cache.h" #include "net/flip/flip_session_pool.h" #include "net/http/http_cache_transaction.h" #include "net/http/http_network_layer.h" #include "net/http/http_network_session.h" #include "net/http/http_request_info.h" #include "net/http/http_response_headers.h" #include "net/http/http_response_info.h" namespace net { // disk cache entry data indices. enum { kResponseInfoIndex, kResponseContentIndex }; //----------------------------------------------------------------------------- HttpCache::ActiveEntry::ActiveEntry(disk_cache::Entry* e) : disk_entry(e), writer(NULL), will_process_pending_queue(false), doomed(false) { } HttpCache::ActiveEntry::~ActiveEntry() { if (disk_entry) disk_entry->Close(); } //----------------------------------------------------------------------------- // This structure keeps track of work items that are attempting to create or // open cache entries. struct HttpCache::NewEntry { NewEntry() : disk_entry(NULL), writer(NULL) {} ~NewEntry() {} disk_cache::Entry* disk_entry; WorkItem* writer; WorkItemList pending_queue; }; //----------------------------------------------------------------------------- // The type of operation represented by a work item. enum WorkItemOperation { WI_OPEN_ENTRY, WI_CREATE_ENTRY, WI_DOOM_ENTRY }; // A work item encapsulates a single request for cache entry with all the // information needed to complete that request. class HttpCache::WorkItem { public: WorkItem(ActiveEntry** entry, CompletionCallback* callback, WorkItemOperation operation) : entry_(entry), callback_(callback), operation_(operation) {} ~WorkItem() {} WorkItemOperation operation() { return operation_; } // Calls back the transaction with the result of the operation. void NotifyTransaction(int result, ActiveEntry* entry) { if (entry_) *entry_ = entry; if (callback_) callback_->Run(result); } void ClearCallback() { callback_ = NULL; } void ClearEntry() { entry_ = NULL; } bool Matches(CompletionCallback* cb) const { return cb == callback_; } bool IsValid() const { return callback_ || entry_; } private: ActiveEntry** entry_; CompletionCallback* callback_; WorkItemOperation operation_; }; //----------------------------------------------------------------------------- // This class is a specialized type of CompletionCallback that allows us to // pass multiple arguments to the completion routine. class HttpCache::BackendCallback : public CallbackRunner<Tuple1<int> > { public: BackendCallback(HttpCache* cache, NewEntry* entry) : cache_(cache), entry_(entry) {} ~BackendCallback() {} virtual void RunWithParams(const Tuple1<int>& params) { cache_->OnIOComplete(params.a, entry_); delete this; } private: HttpCache* cache_; NewEntry* entry_; DISALLOW_COPY_AND_ASSIGN(BackendCallback); }; //----------------------------------------------------------------------------- HttpCache::HttpCache(NetworkChangeNotifier* network_change_notifier, HostResolver* host_resolver, ProxyService* proxy_service, SSLConfigService* ssl_config_service, const FilePath& cache_dir, int cache_size) : disk_cache_dir_(cache_dir), mode_(NORMAL), type_(DISK_CACHE), network_layer_(HttpNetworkLayer::CreateFactory( network_change_notifier, host_resolver, proxy_service, ssl_config_service)), ALLOW_THIS_IN_INITIALIZER_LIST(task_factory_(this)), enable_range_support_(true), cache_size_(cache_size) { } HttpCache::HttpCache(HttpNetworkSession* session, const FilePath& cache_dir, int cache_size) : disk_cache_dir_(cache_dir), mode_(NORMAL), type_(DISK_CACHE), network_layer_(HttpNetworkLayer::CreateFactory(session)), ALLOW_THIS_IN_INITIALIZER_LIST(task_factory_(this)), enable_range_support_(true), cache_size_(cache_size) { } HttpCache::HttpCache(NetworkChangeNotifier* network_change_notifier, HostResolver* host_resolver, ProxyService* proxy_service, SSLConfigService* ssl_config_service, int cache_size) : mode_(NORMAL), type_(MEMORY_CACHE), network_layer_(HttpNetworkLayer::CreateFactory( network_change_notifier, host_resolver, proxy_service, ssl_config_service)), ALLOW_THIS_IN_INITIALIZER_LIST(task_factory_(this)), enable_range_support_(true), cache_size_(cache_size) { } HttpCache::HttpCache(HttpTransactionFactory* network_layer, disk_cache::Backend* disk_cache) : mode_(NORMAL), type_(DISK_CACHE), network_layer_(network_layer), disk_cache_(disk_cache), ALLOW_THIS_IN_INITIALIZER_LIST(task_factory_(this)), enable_range_support_(true), cache_size_(0) { } HttpCache::~HttpCache() { // If we have any active entries remaining, then we need to deactivate them. // We may have some pending calls to OnProcessPendingQueue, but since those // won't run (due to our destruction), we can simply ignore the corresponding // will_process_pending_queue flag. while (!active_entries_.empty()) { ActiveEntry* entry = active_entries_.begin()->second; entry->will_process_pending_queue = false; entry->pending_queue.clear(); entry->readers.clear(); entry->writer = NULL; DeactivateEntry(entry); } ActiveEntriesSet::iterator it = doomed_entries_.begin(); for (; it != doomed_entries_.end(); ++it) delete *it; } disk_cache::Backend* HttpCache::GetBackend() { if (disk_cache_.get()) return disk_cache_.get(); DCHECK_GE(cache_size_, 0); if (type_ == MEMORY_CACHE) { // We may end up with no folder name and no cache if the initialization // of the disk cache fails. We want to be sure that what we wanted to have // was an in-memory cache. disk_cache_.reset(disk_cache::CreateInMemoryCacheBackend(cache_size_)); } else if (!disk_cache_dir_.empty()) { disk_cache_.reset(disk_cache::CreateCacheBackend(disk_cache_dir_, true, cache_size_, type_)); disk_cache_dir_ = FilePath(); // Reclaim memory. } return disk_cache_.get(); } int HttpCache::CreateTransaction(scoped_ptr<HttpTransaction>* trans) { // Do lazy initialization of disk cache if needed. GetBackend(); trans->reset(new HttpCache::Transaction(this, enable_range_support_)); return OK; } HttpCache* HttpCache::GetCache() { return this; } HttpNetworkSession* HttpCache::GetSession() { net::HttpNetworkLayer* network = static_cast<net::HttpNetworkLayer*>(network_layer_.get()); return network->GetSession(); } void HttpCache::Suspend(bool suspend) { network_layer_->Suspend(suspend); } // static bool HttpCache::ParseResponseInfo(const char* data, int len, HttpResponseInfo* response_info, bool* response_truncated) { Pickle pickle(data, len); return response_info->InitFromPickle(pickle, response_truncated); } // static bool HttpCache::ReadResponseInfo(disk_cache::Entry* disk_entry, HttpResponseInfo* response_info, bool* response_truncated) { int size = disk_entry->GetDataSize(kResponseInfoIndex); scoped_refptr<IOBuffer> buffer = new IOBuffer(size); int rv = disk_entry->ReadData(kResponseInfoIndex, 0, buffer, size, NULL); if (rv != size) { DLOG(ERROR) << "ReadData failed: " << rv; return false; } return ParseResponseInfo(buffer->data(), size, response_info, response_truncated); } // static bool HttpCache::WriteResponseInfo(disk_cache::Entry* disk_entry, const HttpResponseInfo* response_info, bool skip_transient_headers, bool response_truncated) { Pickle pickle; response_info->Persist( &pickle, skip_transient_headers, response_truncated); scoped_refptr<WrappedIOBuffer> data = new WrappedIOBuffer( reinterpret_cast<const char*>(pickle.data())); int len = static_cast<int>(pickle.size()); return disk_entry->WriteData(kResponseInfoIndex, 0, data, len, NULL, true) == len; } // Generate a key that can be used inside the cache. std::string HttpCache::GenerateCacheKey(const HttpRequestInfo* request) { // Strip out the reference, username, and password sections of the URL. std::string url = HttpUtil::SpecForRequest(request->url); DCHECK(mode_ != DISABLE); if (mode_ == NORMAL) { // No valid URL can begin with numerals, so we should not have to worry // about collisions with normal URLs. if (request->upload_data && request->upload_data->identifier()) { url.insert(0, StringPrintf("%" PRId64 "/", request->upload_data->identifier())); } return url; } // In playback and record mode, we cache everything. // Lazily initialize. if (playback_cache_map_ == NULL) playback_cache_map_.reset(new PlaybackCacheMap()); // Each time we request an item from the cache, we tag it with a // generation number. During playback, multiple fetches for the same // item will use the same generation number and pull the proper // instance of an URL from the cache. int generation = 0; DCHECK(playback_cache_map_ != NULL); if (playback_cache_map_->find(url) != playback_cache_map_->end()) generation = (*playback_cache_map_)[url]; (*playback_cache_map_)[url] = generation + 1; // The key into the cache is GENERATION # + METHOD + URL. std::string result = IntToString(generation); result.append(request->method); result.append(url); return result; } int HttpCache::DoomEntry(const std::string& key, CompletionCallback* callback) { // Need to abandon the ActiveEntry, but any transaction attached to the entry // should not be impacted. Dooming an entry only means that it will no // longer be returned by FindActiveEntry (and it will also be destroyed once // all consumers are finished with the entry). ActiveEntriesMap::iterator it = active_entries_.find(key); if (it == active_entries_.end()) { return AsyncDoomEntry(key, callback); } ActiveEntry* entry = it->second; active_entries_.erase(it); // We keep track of doomed entries so that we can ensure that they are // cleaned up properly when the cache is destroyed. doomed_entries_.insert(entry); entry->disk_entry->Doom(); entry->doomed = true; DCHECK(entry->writer || !entry->readers.empty()); return OK; } int HttpCache::AsyncDoomEntry(const std::string& key, CompletionCallback* callback) { DCHECK(callback); WorkItem* item = new WorkItem(NULL, callback, WI_DOOM_ENTRY); NewEntry* new_entry = GetNewEntry(key); if (new_entry->writer) { new_entry->pending_queue.push_back(item); return ERR_IO_PENDING; } DCHECK(new_entry->pending_queue.empty()); new_entry->writer = item; BackendCallback* my_callback = new BackendCallback(this, new_entry); int rv = disk_cache_->DoomEntry(key, my_callback); if (rv != ERR_IO_PENDING) { item->ClearCallback(); my_callback->Run(rv); } return rv; } void HttpCache::FinalizeDoomedEntry(ActiveEntry* entry) { DCHECK(entry->doomed); DCHECK(!entry->writer); DCHECK(entry->readers.empty()); DCHECK(entry->pending_queue.empty()); ActiveEntriesSet::iterator it = doomed_entries_.find(entry); DCHECK(it != doomed_entries_.end()); doomed_entries_.erase(it); delete entry; } HttpCache::ActiveEntry* HttpCache::FindActiveEntry(const std::string& key) { ActiveEntriesMap::const_iterator it = active_entries_.find(key); return it != active_entries_.end() ? it->second : NULL; } HttpCache::NewEntry* HttpCache::GetNewEntry(const std::string& key) { DCHECK(!FindActiveEntry(key)); NewEntriesMap::const_iterator it = new_entries_.find(key); if (it != new_entries_.end()) return it->second; NewEntry* entry = new NewEntry(); new_entries_[key] = entry; return entry; } void HttpCache::DeleteNewEntry(NewEntry* entry) { std::string key; if (entry->disk_entry) key = entry->disk_entry->GetKey(); if (!key.empty()) { NewEntriesMap::iterator it = new_entries_.find(key); DCHECK(it != new_entries_.end()); new_entries_.erase(it); } else { for (NewEntriesMap::iterator it = new_entries_.begin(); it != new_entries_.end(); ++it) { if (it->second == entry) { new_entries_.erase(it); break; } } } delete entry; } int HttpCache::OpenEntry(const std::string& key, ActiveEntry** entry, CompletionCallback* callback) { ActiveEntry* active_entry = FindActiveEntry(key); if (active_entry) { *entry = active_entry; return OK; } WorkItem* item = new WorkItem(entry, callback, WI_OPEN_ENTRY); NewEntry* new_entry = GetNewEntry(key); if (new_entry->writer) { new_entry->pending_queue.push_back(item); return ERR_IO_PENDING; } DCHECK(new_entry->pending_queue.empty()); new_entry->writer = item; BackendCallback* my_callback = new BackendCallback(this, new_entry); int rv = disk_cache_->OpenEntry(key, &(new_entry->disk_entry), my_callback); if (rv != ERR_IO_PENDING) { item->ClearCallback(); my_callback->Run(rv); } return rv; } int HttpCache::CreateEntry(const std::string& key, ActiveEntry** entry, CompletionCallback* callback) { DCHECK(!FindActiveEntry(key)); WorkItem* item = new WorkItem(entry, callback, WI_CREATE_ENTRY); NewEntry* new_entry = GetNewEntry(key); if (new_entry->writer) { new_entry->pending_queue.push_back(item); return ERR_IO_PENDING; } DCHECK(new_entry->pending_queue.empty()); new_entry->writer = item; BackendCallback* my_callback = new BackendCallback(this, new_entry); int rv = disk_cache_->CreateEntry(key, &(new_entry->disk_entry), my_callback); if (rv != ERR_IO_PENDING) { item->ClearCallback(); my_callback->Run(rv); } return rv; } void HttpCache::DestroyEntry(ActiveEntry* entry) { if (entry->doomed) { FinalizeDoomedEntry(entry); } else { DeactivateEntry(entry); } } HttpCache::ActiveEntry* HttpCache::ActivateEntry( const std::string& key, disk_cache::Entry* disk_entry) { DCHECK(!FindActiveEntry(key)); ActiveEntry* entry = new ActiveEntry(disk_entry); active_entries_[key] = entry; return entry; } void HttpCache::DeactivateEntry(ActiveEntry* entry) { DCHECK(!entry->will_process_pending_queue); DCHECK(!entry->doomed); DCHECK(!entry->writer); DCHECK(entry->readers.empty()); DCHECK(entry->pending_queue.empty()); std::string key = entry->disk_entry->GetKey(); if (key.empty()) return SlowDeactivateEntry(entry); ActiveEntriesMap::iterator it = active_entries_.find(key); DCHECK(it != active_entries_.end()); DCHECK(it->second == entry); active_entries_.erase(it); delete entry; } // We don't know this entry's key so we have to find it without it. void HttpCache::SlowDeactivateEntry(ActiveEntry* entry) { for (ActiveEntriesMap::iterator it = active_entries_.begin(); it != active_entries_.end(); ++it) { if (it->second == entry) { active_entries_.erase(it); delete entry; break; } } } int HttpCache::AddTransactionToEntry(ActiveEntry* entry, Transaction* trans) { DCHECK(entry); // We implement a basic reader/writer lock for the disk cache entry. If // there is already a writer, then everyone has to wait for the writer to // finish before they can access the cache entry. There can be multiple // readers. // // NOTE: If the transaction can only write, then the entry should not be in // use (since any existing entry should have already been doomed). if (entry->writer || entry->will_process_pending_queue) { entry->pending_queue.push_back(trans); return ERR_IO_PENDING; } if (trans->mode() & Transaction::WRITE) { // transaction needs exclusive access to the entry if (entry->readers.empty()) { entry->writer = trans; } else { entry->pending_queue.push_back(trans); return ERR_IO_PENDING; } } else { // transaction needs read access to the entry entry->readers.push_back(trans); } // We do this before calling EntryAvailable to force any further calls to // AddTransactionToEntry to add their transaction to the pending queue, which // ensures FIFO ordering. if (!entry->writer && !entry->pending_queue.empty()) ProcessPendingQueue(entry); return trans->EntryAvailable(entry); } void HttpCache::DoneWithEntry(ActiveEntry* entry, Transaction* trans, bool cancel) { // If we already posted a task to move on to the next transaction and this was // the writer, there is nothing to cancel. if (entry->will_process_pending_queue && entry->readers.empty()) return; if (entry->writer) { DCHECK(trans == entry->writer); // Assume there was a failure. bool success = false; if (cancel) { DCHECK(entry->disk_entry); // This is a successful operation in the sense that we want to keep the // entry. success = trans->AddTruncatedFlag(); } DoneWritingToEntry(entry, success); } else { DoneReadingFromEntry(entry, trans); } } void HttpCache::DoneWritingToEntry(ActiveEntry* entry, bool success) { DCHECK(entry->readers.empty()); entry->writer = NULL; if (success) { ProcessPendingQueue(entry); } else { DCHECK(!entry->will_process_pending_queue); // We failed to create this entry. TransactionList pending_queue; pending_queue.swap(entry->pending_queue); entry->disk_entry->Doom(); DestroyEntry(entry); // We need to do something about these pending entries, which now need to // be added to a new entry. while (!pending_queue.empty()) { pending_queue.front()->AddToEntry(); pending_queue.pop_front(); } } } void HttpCache::DoneReadingFromEntry(ActiveEntry* entry, Transaction* trans) { DCHECK(!entry->writer); TransactionList::iterator it = std::find(entry->readers.begin(), entry->readers.end(), trans); DCHECK(it != entry->readers.end()); entry->readers.erase(it); ProcessPendingQueue(entry); } void HttpCache::ConvertWriterToReader(ActiveEntry* entry) { DCHECK(entry->writer); DCHECK(entry->writer->mode() == Transaction::READ_WRITE); DCHECK(entry->readers.empty()); Transaction* trans = entry->writer; entry->writer = NULL; entry->readers.push_back(trans); ProcessPendingQueue(entry); } void HttpCache::RemovePendingTransaction(Transaction* trans, CompletionCallback* cb) { ActiveEntriesMap::const_iterator i = active_entries_.find(trans->key()); bool found = false; if (i != active_entries_.end()) found = RemovePendingTransactionFromEntry(i->second, trans); if (found) return; NewEntriesMap::const_iterator j = new_entries_.find(trans->key()); if (j != new_entries_.end()) found = RemovePendingCallbackFromNewEntry(j->second, cb); ActiveEntriesSet::iterator k = doomed_entries_.begin(); for (; k != doomed_entries_.end() && !found; ++k) found = RemovePendingTransactionFromEntry(*k, trans); DCHECK(found) << "Pending transaction not found"; } bool HttpCache::RemovePendingTransactionFromEntry(ActiveEntry* entry, Transaction* trans) { TransactionList& pending_queue = entry->pending_queue; TransactionList::iterator j = find(pending_queue.begin(), pending_queue.end(), trans); if (j == pending_queue.end()) return false; pending_queue.erase(j); return true; } bool HttpCache::RemovePendingCallbackFromNewEntry(NewEntry* entry, CompletionCallback* cb) { if (entry->writer->Matches(cb)) { entry->writer->ClearCallback(); entry->writer->ClearEntry(); return true; } WorkItemList& pending_queue = entry->pending_queue; WorkItemList::iterator it = pending_queue.begin(); for (; it != pending_queue.end(); ++it) { if ((*it)->Matches(cb)) { delete *it; pending_queue.erase(it); return true; } } return false; } void HttpCache::ProcessPendingQueue(ActiveEntry* entry) { // Multiple readers may finish with an entry at once, so we want to batch up // calls to OnProcessPendingQueue. This flag also tells us that we should // not delete the entry before OnProcessPendingQueue runs. if (entry->will_process_pending_queue) return; entry->will_process_pending_queue = true; MessageLoop::current()->PostTask(FROM_HERE, task_factory_.NewRunnableMethod(&HttpCache::OnProcessPendingQueue, entry)); } void HttpCache::OnProcessPendingQueue(ActiveEntry* entry) { entry->will_process_pending_queue = false; DCHECK(!entry->writer); // If no one is interested in this entry, then we can de-activate it. if (entry->pending_queue.empty()) { if (entry->readers.empty()) DestroyEntry(entry); return; } // Promote next transaction from the pending queue. Transaction* next = entry->pending_queue.front(); if ((next->mode() & Transaction::WRITE) && !entry->readers.empty()) return; // Have to wait. entry->pending_queue.erase(entry->pending_queue.begin()); AddTransactionToEntry(entry, next); } void HttpCache::OnIOComplete(int result, NewEntry* new_entry) { scoped_ptr<WorkItem> item(new_entry->writer); WorkItemOperation op = item->operation(); bool fail_requests = false; ActiveEntry* entry = NULL; std::string key; if (result == OK) { if (op == WI_DOOM_ENTRY) { // Anything after a Doom has to be restarted. fail_requests = true; } else if (item->IsValid()) { key = new_entry->disk_entry->GetKey(); entry = ActivateEntry(key, new_entry->disk_entry); } else { // The writer transaction is gone. if (op == WI_CREATE_ENTRY) new_entry->disk_entry->Doom(); new_entry->disk_entry->Close(); fail_requests = true; } } // We are about to notify a bunch of transactions, and they may decide to // re-issue a request (or send a different one). If we don't delete new_entry, // the new request will be appended to the end of the list, and we'll see it // again from this point before it has a chance to complete (and we'll be // messing out the request order). The down side is that if for some reason // notifying request A ends up cancelling request B (for the same key), we // won't find request B anywhere (because it would be in a local variable // here) and that's bad. If there is a chance for that to happen, we'll have // to move the callback used to be a CancelableCallback. By the way, for this // to happen the action (to cancel B) has to be synchronous to the // notification for request A. WorkItemList pending_items; pending_items.swap(new_entry->pending_queue); DeleteNewEntry(new_entry); item->NotifyTransaction(result, entry); while (!pending_items.empty()) { item.reset(pending_items.front()); pending_items.pop_front(); if (item->operation() == WI_DOOM_ENTRY) { // A queued doom request is always a race. fail_requests = true; } else if (result == OK) { entry = FindActiveEntry(key); if (!entry) fail_requests = true; } if (fail_requests) { item->NotifyTransaction(ERR_CACHE_RACE, NULL); continue; } if (item->operation() == WI_CREATE_ENTRY) { if (result == OK) { // A second Create request, but the first request succeded. item->NotifyTransaction(ERR_CACHE_CREATE_FAILURE, NULL); } else { if (op != WI_CREATE_ENTRY) { // Failed Open followed by a Create. item->NotifyTransaction(ERR_CACHE_RACE, NULL); fail_requests = true; } else { item->NotifyTransaction(result, entry); } } } else { if (op == WI_CREATE_ENTRY && result != OK) { // Failed Create followed by an Open. item->NotifyTransaction(ERR_CACHE_RACE, NULL); fail_requests = true; } else { item->NotifyTransaction(result, entry); } } } } void HttpCache::CloseCurrentConnections() { net::HttpNetworkLayer* network = static_cast<net::HttpNetworkLayer*>(network_layer_.get()); HttpNetworkSession* session = network->GetSession(); if (session) { session->tcp_socket_pool()->CloseIdleSockets(); if (session->flip_session_pool()) session->flip_session_pool()->CloseAllSessions(); session->ReplaceTCPSocketPool(); } } //----------------------------------------------------------------------------- } // namespace net