// Copyright (c) 2012 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. // // This file contains the Search autocomplete provider. This provider is // responsible for all autocomplete entries that start with "Search <engine> // for ...", including searching for the current input string, search // history, and search suggestions. An instance of it gets created and // managed by the autocomplete controller. #ifndef CHROME_BROWSER_AUTOCOMPLETE_SEARCH_PROVIDER_H_ #define CHROME_BROWSER_AUTOCOMPLETE_SEARCH_PROVIDER_H_ #include <map> #include <string> #include <vector> #include "base/basictypes.h" #include "base/compiler_specific.h" #include "base/memory/scoped_ptr.h" #include "base/memory/scoped_vector.h" #include "base/time/time.h" #include "base/timer/timer.h" #include "chrome/browser/autocomplete/autocomplete_input.h" #include "chrome/browser/autocomplete/autocomplete_match.h" #include "chrome/browser/autocomplete/autocomplete_provider.h" #include "chrome/browser/history/history_types.h" #include "chrome/browser/search_engines/template_url.h" #include "net/url_request/url_fetcher_delegate.h" class Profile; class SearchProviderTest; class SuggestionDeletionHandler; class TemplateURLService; namespace base { class Value; } namespace net { class URLFetcher; } // Autocomplete provider for searches and suggestions from a search engine. // // After construction, the autocomplete controller repeatedly calls Start() // with some user input, each time expecting to receive a small set of the best // matches (either synchronously or asynchronously). // // Initially the provider creates a match that searches for the current input // text. It also starts a task to query the Suggest servers. When that data // comes back, the provider creates and returns matches for the best // suggestions. class SearchProvider : public AutocompleteProvider, public net::URLFetcherDelegate { public: // ID used in creating URLFetcher for default provider's suggest results. static const int kDefaultProviderURLFetcherID; // ID used in creating URLFetcher for keyword provider's suggest results. static const int kKeywordProviderURLFetcherID; // ID used in creating URLFetcher for deleting suggestion results. static const int kDeletionURLFetcherID; SearchProvider(AutocompleteProviderListener* listener, Profile* profile); // Returns an AutocompleteMatch with the given |autocomplete_provider|, // |relevance|, and |type|, which represents a search via |template_url| for // |query_string|. If |template_url| is NULL, returns a match with an invalid // destination URL. // // |input_text| is the original user input, which may differ from // |query_string|; e.g. the user typed "foo" and got a search suggestion of // "food", which we're now marking up. This is used to highlight portions of // the match contents to distinguish locally-typed text from suggested text. // // |input| and |is_keyword| are necessary for various other details, like // whether we should allow inline autocompletion and what the transition type // should be. |accepted_suggestion| and |omnibox_start_margin| are used along // with |input_text| to generate Assisted Query Stats. // |append_extra_query_params| should be set if |template_url| is the default // search engine, so the destination URL will contain any // command-line-specified query params. static AutocompleteMatch CreateSearchSuggestion( AutocompleteProvider* autocomplete_provider, const AutocompleteInput& input, const base::string16& input_text, int relevance, AutocompleteMatch::Type type, bool is_keyword, const base::string16& match_contents, const base::string16& annotation, const TemplateURL* template_url, const base::string16& query_string, const std::string& suggest_query_params, int accepted_suggestion, int omnibox_start_margin, bool append_extra_query_params); // Returns whether the SearchProvider previously flagged |match| as a query // that should be prefetched. static bool ShouldPrefetch(const AutocompleteMatch& match); // Extracts the suggest response metadata which SearchProvider previously // stored for |match|. static std::string GetSuggestMetadata(const AutocompleteMatch& match); // AutocompleteProvider: virtual void AddProviderInfo(ProvidersInfo* provider_info) const OVERRIDE; virtual void DeleteMatch(const AutocompleteMatch& match) OVERRIDE; virtual void ResetSession() OVERRIDE; bool field_trial_triggered_in_session() const { return field_trial_triggered_in_session_; } // This URL may be sent with suggest requests; see comments on CanSendURL(). void set_current_page_url(const GURL& current_page_url) { current_page_url_ = current_page_url; } protected: virtual ~SearchProvider(); private: // TODO(hfung): Remove ZeroSuggestProvider as a friend class after // refactoring common code to a new base class. friend class SearchProviderTest; friend class ZeroSuggestProvider; FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, CanSendURL); FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, NavigationInline); FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, NavigationInlineDomainClassify); FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, NavigationInlineSchemeSubstring); FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, RemoveStaleResultsTest); FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, SuggestRelevanceExperiment); FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, TestDeleteMatch); FRIEND_TEST_ALL_PREFIXES(AutocompleteProviderTest, GetDestinationURL); FRIEND_TEST_ALL_PREFIXES(InstantExtendedPrefetchTest, ClearPrefetchedResults); FRIEND_TEST_ALL_PREFIXES(InstantExtendedPrefetchTest, SetPrefetchQuery); // Manages the providers (TemplateURLs) used by SearchProvider. Two providers // may be used: // . The default provider. This corresponds to the user's default search // engine. This is always used, except for the rare case of no default // engine. // . The keyword provider. This is used if the user has typed in a keyword. class Providers { public: explicit Providers(TemplateURLService* template_url_service); // Returns true if the specified providers match the two providers cached // by this class. bool equal(const base::string16& default_provider, const base::string16& keyword_provider) const { return (default_provider == default_provider_) && (keyword_provider == keyword_provider_); } // Resets the cached providers. void set(const base::string16& default_provider, const base::string16& keyword_provider) { default_provider_ = default_provider; keyword_provider_ = keyword_provider; } TemplateURLService* template_url_service() { return template_url_service_; } const base::string16& default_provider() const { return default_provider_; } const base::string16& keyword_provider() const { return keyword_provider_; } // NOTE: These may return NULL even if the provider members are nonempty! const TemplateURL* GetDefaultProviderURL() const; const TemplateURL* GetKeywordProviderURL() const; // Returns true if there is a valid keyword provider. bool has_keyword_provider() const { return !keyword_provider_.empty(); } private: TemplateURLService* template_url_service_; // Cached across the life of a query so we behave consistently even if the // user changes their default while the query is running. base::string16 default_provider_; base::string16 keyword_provider_; DISALLOW_COPY_AND_ASSIGN(Providers); }; // The Result classes are intermediate representations of AutocompleteMatches, // simply containing relevance-ranked search and navigation suggestions. // They may be cached to provide some synchronous matches while requests for // new suggestions from updated input are in flight. // TODO(msw) Extend these classes to generate their corresponding matches and // other requisite data, in order to consolidate and simplify the // highly fragmented SearchProvider logic for each Result type. class Result { public: Result(bool from_keyword_provider, int relevance, bool relevance_from_server); virtual ~Result(); bool from_keyword_provider() const { return from_keyword_provider_; } int relevance() const { return relevance_; } void set_relevance(int relevance) { relevance_ = relevance; } bool relevance_from_server() const { return relevance_from_server_; } void set_relevance_from_server(bool relevance_from_server) { relevance_from_server_ = relevance_from_server; } // Returns if this result is inlineable against the current input |input|. // Non-inlineable results are stale. virtual bool IsInlineable(const base::string16& input) const = 0; // Returns the default relevance value for this result (which may // be left over from a previous omnibox input) given the current // input and whether the current input caused a keyword provider // to be active. virtual int CalculateRelevance(const AutocompleteInput& input, bool keyword_provider_requested) const = 0; protected: // True if the result came from the keyword provider. bool from_keyword_provider_; // The relevance score. int relevance_; private: // Whether this result's relevance score was fully or partly calculated // based on server information, and thus is assumed to be more accurate. // This is ultimately used in // SearchProvider::ConvertResultsToAutocompleteMatches(), see comments // there. bool relevance_from_server_; }; class SuggestResult : public Result { public: SuggestResult(const base::string16& suggestion, AutocompleteMatchType::Type type, const base::string16& match_contents, const base::string16& annotation, const std::string& suggest_query_params, const std::string& deletion_url, bool from_keyword_provider, int relevance, bool relevance_from_server, bool should_prefetch); virtual ~SuggestResult(); const base::string16& suggestion() const { return suggestion_; } AutocompleteMatchType::Type type() const { return type_; } const base::string16& match_contents() const { return match_contents_; } const base::string16& annotation() const { return annotation_; } const std::string& suggest_query_params() const { return suggest_query_params_; } const std::string& deletion_url() const { return deletion_url_; } bool should_prefetch() const { return should_prefetch_; } // Result: virtual bool IsInlineable(const base::string16& input) const OVERRIDE; virtual int CalculateRelevance( const AutocompleteInput& input, bool keyword_provider_requested) const OVERRIDE; private: // The search terms to be used for this suggestion. base::string16 suggestion_; AutocompleteMatchType::Type type_; // The contents to be displayed in the autocomplete match. base::string16 match_contents_; // Optional annotation for the |match_contents_| for disambiguation. // This may be displayed in the autocomplete match contents, but is defined // separately to facilitate different formatting. base::string16 annotation_; // Optional additional parameters to be added to the search URL. std::string suggest_query_params_; // Optional deletion URL provided with suggestions. Fetching this URL // should result in some reasonable deletion behaviour on the server, // e.g. deleting this term out of a user's server-side search history. std::string deletion_url_; // Should this result be prefetched? bool should_prefetch_; }; class NavigationResult : public Result { public: // |provider| is necessary to use StringForURLDisplay() in order to // compute |formatted_url_|. NavigationResult(const AutocompleteProvider& provider, const GURL& url, const base::string16& description, bool from_keyword_provider, int relevance, bool relevance_from_server); virtual ~NavigationResult(); const GURL& url() const { return url_; } const base::string16& description() const { return description_; } const base::string16& formatted_url() const { return formatted_url_; } // Result: virtual bool IsInlineable(const base::string16& input) const OVERRIDE; virtual int CalculateRelevance( const AutocompleteInput& input, bool keyword_provider_requested) const OVERRIDE; private: // The suggested url for navigation. GURL url_; // The properly formatted ("fixed up") URL string with equivalent meaning // to the one in |url_|. base::string16 formatted_url_; // The suggested navigational result description; generally the site name. base::string16 description_; }; class CompareScoredResults; typedef std::vector<SuggestResult> SuggestResults; typedef std::vector<NavigationResult> NavigationResults; typedef std::vector<history::KeywordSearchTermVisit> HistoryResults; typedef std::pair<base::string16, std::string> MatchKey; typedef std::map<MatchKey, AutocompleteMatch> MatchMap; typedef ScopedVector<SuggestionDeletionHandler> SuggestionDeletionHandlers; // A simple structure bundling most of the information (including // both SuggestResults and NavigationResults) returned by a call to // the suggest server. // // This has to be declared after the typedefs since it relies on some of them. struct Results { Results(); ~Results(); // Clears |suggest_results| and |navigation_results| and resets // |verbatim_relevance| to -1 (implies unset). void Clear(); // Returns whether any of the results (including verbatim) have // server-provided scores. bool HasServerProvidedScores() const; // Query suggestions sorted by relevance score. SuggestResults suggest_results; // Navigational suggestions sorted by relevance score. NavigationResults navigation_results; // The server supplied verbatim relevance scores. Negative values // indicate that there is no suggested score; a value of 0 // suppresses the verbatim result. int verbatim_relevance; // The JSON metadata associated with this server response. std::string metadata; private: DISALLOW_COPY_AND_ASSIGN(Results); }; // Removes non-inlineable results until either the top result can inline // autocomplete the current input or verbatim outscores the top result. static void RemoveStaleResults(const base::string16& input, int verbatim_relevance, SuggestResults* suggest_results, NavigationResults* navigation_results); // Calculates the relevance score for the keyword verbatim result (if the // input matches one of the profile's keyword). static int CalculateRelevanceForKeywordVerbatim(AutocompleteInput::Type type, bool prefer_keyword); // AutocompleteProvider: virtual void Start(const AutocompleteInput& input, bool minimal_changes) OVERRIDE; virtual void Stop(bool clear_cached_results) OVERRIDE; // net::URLFetcherDelegate: virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE; // This gets called when we have requested a suggestion deletion from the // server to handle the results of the deletion. void OnDeletionComplete(bool success, SuggestionDeletionHandler* handler); // Records in UMA whether the deletion request resulted in success. // This is virtual so test code can override it to check that we // correctly handle the request result. virtual void RecordDeletionResult(bool success); // Removes the deleted match from the list of |matches_|. void DeleteMatchFromMatches(const AutocompleteMatch& match); // Called when timer_ expires. void Run(); // Runs the history query, if necessary. The history query is synchronous. // This does not update |done_|. void DoHistoryQuery(bool minimal_changes); // Determines whether an asynchronous subcomponent query should run for the // current input. If so, starts it if necessary; otherwise stops it. // NOTE: This function does not update |done_|. Callers must do so. void StartOrStopSuggestQuery(bool minimal_changes); // Returns true when the current query can be sent to the Suggest service. // This will be false e.g. when Suggest is disabled, the query contains // potentially private data, etc. bool IsQuerySuitableForSuggest() const; // Stops the suggest query. // NOTE: This does not update |done_|. Callers must do so. void StopSuggest(); // Clears the current results. void ClearAllResults(); // Removes stale results for both default and keyword providers. See comments // on RemoveStaleResults(). void RemoveAllStaleResults(); // Apply calculated relevance scores to the current results. void ApplyCalculatedRelevance(); void ApplyCalculatedSuggestRelevance(SuggestResults* list); void ApplyCalculatedNavigationRelevance(NavigationResults* list); // Starts a new URLFetcher requesting suggest results from |template_url|; // callers own the returned URLFetcher, which is NULL for invalid providers. net::URLFetcher* CreateSuggestFetcher(int id, const TemplateURL* template_url, const AutocompleteInput& input); // Parses JSON response received from the provider, stripping XSSI // protection if needed. Returns the parsed data if successful, NULL // otherwise. static scoped_ptr<base::Value> DeserializeJsonData(std::string json_data); // Parses results from the suggest server and updates the appropriate suggest // and navigation result lists, depending on whether |is_keyword| is true. // Returns whether the appropriate result list members were updated. bool ParseSuggestResults(base::Value* root_val, bool is_keyword); // Converts the parsed results to a set of AutocompleteMatches, |matches_|. void ConvertResultsToAutocompleteMatches(); // Returns an iterator to the first match in |matches_| which might // be chosen as default. If // |autocomplete_result_will_reorder_for_default_match| is false, // this simply means the first match; otherwise, it means the first // match for which the |allowed_to_be_default_match| member is true. ACMatches::const_iterator FindTopMatch( bool autocomplete_result_will_reorder_for_default_match) const; // Checks if suggested relevances violate certain expected constraints. // See UpdateMatches() for the use and explanation of these constraints. bool IsTopMatchNavigationInKeywordMode( bool autocomplete_result_will_reorder_for_default_match) const; bool HasKeywordDefaultMatchInKeywordMode() const; bool IsTopMatchScoreTooLow( bool autocomplete_result_will_reorder_for_default_match) const; bool IsTopMatchSearchWithURLInput( bool autocomplete_result_will_reorder_for_default_match) const; bool HasValidDefaultMatch( bool autocomplete_result_will_reorder_for_default_match) const; // Updates |matches_| from the latest results; applies calculated relevances // if suggested relevances cause undesriable behavior. Updates |done_|. void UpdateMatches(); // Converts an appropriate number of navigation results in // |navigation_results| to matches and adds them to |matches|. void AddNavigationResultsToMatches( const NavigationResults& navigation_results, ACMatches* matches); // Adds a match for each result in |results| to |map|. |is_keyword| indicates // whether the results correspond to the keyword provider or default provider. void AddHistoryResultsToMap(const HistoryResults& results, bool is_keyword, int did_not_accept_suggestion, MatchMap* map); // Calculates relevance scores for all |results|. SuggestResults ScoreHistoryResults(const HistoryResults& results, bool base_prevent_inline_autocomplete, bool input_multiple_words, const base::string16& input_text, bool is_keyword); // Adds matches for |results| to |map|. void AddSuggestResultsToMap(const SuggestResults& results, const std::string& metadata, MatchMap* map); // Gets the relevance score for the verbatim result. This value may be // provided by the suggest server or calculated locally; if // |relevance_from_server| is non-NULL, it will be set to indicate which of // those is true. int GetVerbatimRelevance(bool* relevance_from_server) const; // Calculates the relevance score for the verbatim result from the // default search engine. This version takes into account context: // i.e., whether the user has entered a keyword-based search or not. int CalculateRelevanceForVerbatim() const; // Calculates the relevance score for the verbatim result from the default // search engine *ignoring* whether the input is a keyword-based search // or not. This function should only be used to determine the minimum // relevance score that the best result from this provider should have. // For normal use, prefer the above function. int CalculateRelevanceForVerbatimIgnoringKeywordModeState() const; // Gets the relevance score for the keyword verbatim result. // |relevance_from_server| is handled as in GetVerbatimRelevance(). // TODO(mpearson): Refactor so this duplication isn't necessary or // restructure so one static function takes all the parameters it needs // (rather than looking at internal state). int GetKeywordVerbatimRelevance(bool* relevance_from_server) const; // |time| is the time at which this query was last seen. |is_keyword| // indicates whether the results correspond to the keyword provider or default // provider. |use_aggressive_method| says whether this function can use a // method that gives high scores (1200+) rather than one that gives lower // scores. When using the aggressive method, scores may exceed 1300 // unless |prevent_search_history_inlining| is set. int CalculateRelevanceForHistory(const base::Time& time, bool is_keyword, bool use_aggressive_method, bool prevent_search_history_inlining) const; // Creates an AutocompleteMatch for "Search <engine> for |query_string|" with // the supplied details. Adds this match to |map|; if such a match already // exists, whichever one has lower relevance is eliminated. void AddMatchToMap(const SuggestResult& result, const base::string16& input_text, const std::string& metadata, int accepted_suggestion, MatchMap* map); // Returns an AutocompleteMatch for a navigational suggestion. AutocompleteMatch NavigationToMatch(const NavigationResult& navigation); // Resets the scores of all |keyword_navigation_results_| matches to // be below that of the top keyword query match (the verbatim match // as expressed by |keyword_verbatim_relevance_| or keyword query // suggestions stored in |keyword_suggest_results_|). If there // are no keyword suggestions and keyword verbatim is suppressed, // then drops the suggested relevance scores for the navsuggestions // and drops the request to suppress verbatim, thereby introducing the // keyword verbatim match which will naturally outscore the navsuggestions. void DemoteKeywordNavigationMatchesPastTopQuery(); // Updates the value of |done_| from the internal state. void UpdateDone(); // Returns whether we can send the URL of the current page in any suggest // requests. Doing this requires that all the following hold: // * The user has suggest enabled in their settings and is not in incognito // mode. (Incognito disables suggest entirely.) // * The current URL is HTTP, or HTTPS with the same domain as the suggest // server. Non-HTTP[S] URLs (e.g. FTP/file URLs) may contain sensitive // information. HTTPS URLs may also contain sensitive information, but if // they're on the same domain as the suggest server, then the relevant // entity could have already seen/logged this data. // * The suggest request is sent over HTTPS. This avoids leaking the current // page URL in world-readable network traffic. // * The user's suggest provider is Google. We might want to allow other // providers to see this data someday, but for now this has only been // implemented for Google. Also see next bullet. // * The user is OK in principle with sending URLs of current pages to their // provider. Today, there is no explicit setting that controls this, but if // the user has tab sync enabled and tab sync is unencrypted, then they're // already sending this data to Google for sync purposes. Thus we use this // setting as a proxy for "it's OK to send such data". In the future, // especially if we want to support suggest providers other than Google, we // may change this to be a standalone setting or part of some explicit // general opt-in. static bool CanSendURL( const GURL& current_page_url, const GURL& suggest_url, const TemplateURL* template_url, AutocompleteInput::PageClassification page_classification, Profile* profile); // The amount of time to wait before sending a new suggest request after the // previous one. Non-const because some unittests modify this value. static int kMinimumTimeBetweenSuggestQueriesMs; // The following keys are used to record additional information on matches. // We annotate our AutocompleteMatches with whether their relevance scores // were server-provided using this key in the |additional_info| field. static const char kRelevanceFromServerKey[]; // Indicates whether the server said a match should be prefetched. static const char kShouldPrefetchKey[]; // Used to store metadata from the server response, which is needed for // prefetching. static const char kSuggestMetadataKey[]; // Used to store a deletion request url for server-provided suggestions. static const char kDeletionUrlKey[]; // These are the values for the above keys. static const char kTrue[]; static const char kFalse[]; // Maintains the TemplateURLs used. Providers providers_; // The user's input. AutocompleteInput input_; // Input when searching against the keyword provider. AutocompleteInput keyword_input_; // Searches in the user's history that begin with the input text. HistoryResults keyword_history_results_; HistoryResults default_history_results_; // Number of suggest results that haven't yet arrived. If greater than 0 it // indicates one of the URLFetchers is still running. int suggest_results_pending_; // A timer to start a query to the suggest server after the user has stopped // typing for long enough. base::OneShotTimer<SearchProvider> timer_; // The time at which we sent a query to the suggest server. base::TimeTicks time_suggest_request_sent_; // Fetchers used to retrieve results for the keyword and default providers. scoped_ptr<net::URLFetcher> keyword_fetcher_; scoped_ptr<net::URLFetcher> default_fetcher_; // Results from the default and keyword search providers. Results default_results_; Results keyword_results_; // Each deletion handler in this vector corresponds to an outstanding request // that a server delete a personalized suggestion. Making this a ScopedVector // causes us to auto-cancel all such requests on shutdown. SuggestionDeletionHandlers deletion_handlers_; // Whether a field trial, if any, has triggered in the most recent // autocomplete query. This field is set to false in Start() and may be set // to true if either the default provider or keyword provider has completed // and their corresponding suggest response contained // '"google:fieldtrialtriggered":true'. // If the autocomplete query has not returned, this field is set to false. bool field_trial_triggered_; // Same as above except that it is maintained across the current Omnibox // session. bool field_trial_triggered_in_session_; // If true, search history query suggestions will score low enough that // they will not be inlined. bool prevent_search_history_inlining_; GURL current_page_url_; DISALLOW_COPY_AND_ASSIGN(SearchProvider); }; #endif // CHROME_BROWSER_AUTOCOMPLETE_SEARCH_PROVIDER_H_