/* * Copyright (C) 2015 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef ART_CMDLINE_CMDLINE_PARSER_H_ #define ART_CMDLINE_CMDLINE_PARSER_H_ #define CMDLINE_NDEBUG 1 // Do not output any debugging information for parsing. #include #include #include #include #include #include #include #include #include "base/indenter.h" #include "base/variant_map.h" #include "cmdline_parse_result.h" #include "cmdline_result.h" #include "cmdline_type_parser.h" #include "cmdline_types.h" #include "detail/cmdline_debug_detail.h" #include "detail/cmdline_parse_argument_detail.h" #include "detail/cmdline_parser_detail.h" #include "token_range.h" namespace art { // Build a parser for command line arguments with a small domain specific language. // Each parsed type must have a specialized CmdlineType in order to do the string->T parsing. // Each argument must also have a VariantMap::Key in order to do the T storage. template class TVariantMapKey> struct CmdlineParser { template struct ArgumentBuilder; struct Builder; // Build the parser. struct UntypedArgumentBuilder; // Build arguments which weren't yet given a type. private: // Forward declare some functions that we need to use before fully-defining structs. template static ArgumentBuilder CreateArgumentBuilder(Builder& parent); static void AppendCompletedArgument(Builder& builder, detail::CmdlineParseArgumentAny* arg); // Allow argument definitions to save their values when they are parsed, // without having a dependency on CmdlineParser or any of the builders. // // A shared pointer to the save destination is saved into the load/save argument callbacks. // // This also allows the underlying storage (i.e. a variant map) to be released // to the user, without having to recreate all of the callbacks. struct SaveDestination { SaveDestination() : variant_map_(new TVariantMap()) {} // Save value to the variant map. template void SaveToMap(const TVariantMapKey& key, TArg& value) { variant_map_->Set(key, value); } // Get the existing value from a map, creating the value if it did not already exist. template TArg& GetOrCreateFromMap(const TVariantMapKey& key) { auto* ptr = variant_map_->Get(key); if (ptr == nullptr) { variant_map_->Set(key, TArg()); ptr = variant_map_->Get(key); assert(ptr != nullptr); } return *ptr; } protected: // Release the map, clearing it as a side-effect. // Future saves will be distinct from previous saves. TVariantMap&& ReleaseMap() { return std::move(*variant_map_); } // Get a read-only reference to the variant map. const TVariantMap& GetMap() { return *variant_map_; } // Clear all potential save targets. void Clear() { variant_map_->Clear(); } private: // Don't try to copy or move this. Just don't. SaveDestination(const SaveDestination&) = delete; SaveDestination(SaveDestination&&) = delete; SaveDestination& operator=(const SaveDestination&) = delete; SaveDestination& operator=(SaveDestination&&) = delete; std::shared_ptr variant_map_; // Allow the parser to change the underlying pointers when we release the underlying storage. friend struct CmdlineParser; }; public: // Builder for the argument definition of type TArg. Do not use this type directly, // it is only a separate type to provide compile-time enforcement against doing // illegal builds. template struct ArgumentBuilder { // Add a range check to this argument. ArgumentBuilder& WithRange(const TArg& min, const TArg& max) { argument_info_.has_range_ = true; argument_info_.min_ = min; argument_info_.max_ = max; return *this; } // Map the list of names into the list of values. List of names must not have // any wildcards '_' in it. // // Do not use if a value map has already been set. ArgumentBuilder& WithValues(std::initializer_list value_list) { SetValuesInternal(value_list); return *this; } // When used with a single alias, map the alias into this value. // Same as 'WithValues({value})' , but allows the omission of the curly braces {}. ArgumentBuilder WithValue(const TArg& value) { return WithValues({ value }); } // Map the parsed string values (from _) onto a concrete value. If no wildcard // has been specified, then map the value directly from the arg name (i.e. // if there are multiple aliases, then use the alias to do the mapping). // // Do not use if a values list has already been set. ArgumentBuilder& WithValueMap( std::initializer_list> key_value_list) { assert(!argument_info_.has_value_list_); argument_info_.has_value_map_ = true; argument_info_.value_map_ = key_value_list; return *this; } // If this argument is seen multiple times, successive arguments mutate the same value // instead of replacing it with a new value. ArgumentBuilder& AppendValues() { argument_info_.appending_values_ = true; return *this; } ArgumentBuilder& WithMetavar(const char* sv) { argument_info_.metavar_ = sv; return *this; } ArgumentBuilder& WithHelp(const char* sv) { argument_info_.help_ = sv; return *this; } // Convenience type alias for the variant map key type definition. using MapKey = TVariantMapKey; // Write the results of this argument into the key. // To look up the parsed arguments, get the map and then use this key with VariantMap::Get CmdlineParser::Builder& IntoKey(const MapKey& key) { // Only capture save destination as a pointer. // This allows the parser to later on change the specific save targets. auto save_destination = save_destination_; save_value_ = [save_destination, &key](TArg& value) { save_destination->SaveToMap(key, value); CMDLINE_DEBUG_LOG << "Saved value into map '" << detail::ToStringAny(value) << "'" << std::endl; }; load_value_ = [save_destination, &key]() -> TArg& { TArg& value = save_destination->GetOrCreateFromMap(key); CMDLINE_DEBUG_LOG << "Loaded value from map '" << detail::ToStringAny(value) << "'" << std::endl; return value; }; save_value_specified_ = true; load_value_specified_ = true; CompleteArgument(); return parent_; } // Write the results of this argument into a variable pointed to by destination. // An optional is used to tell whether the command line argument was present. CmdlineParser::Builder& IntoLocation(std::optional* destination) { save_value_ = [destination](TArg& value) { *destination = value; }; load_value_ = [destination]() -> TArg& { return destination->value(); }; save_value_specified_ = true; load_value_specified_ = true; CompleteArgument(); return parent_; } // Ensure we always move this when returning a new builder. ArgumentBuilder(ArgumentBuilder&&) noexcept = default; protected: // Used by builder to internally ignore arguments by dropping them on the floor after parsing. CmdlineParser::Builder& IntoIgnore() { save_value_ = [](TArg& value) { CMDLINE_DEBUG_LOG << "Ignored value '" << detail::ToStringAny(value) << "'" << std::endl; }; load_value_ = []() -> TArg& { assert(false && "Should not be appending values to ignored arguments"); __builtin_trap(); // Blow up. }; save_value_specified_ = true; load_value_specified_ = true; CompleteArgument(); return parent_; } void SetValuesInternal(const std::vector&& value_list) { assert(!argument_info_.has_value_map_); argument_info_.has_value_list_ = true; argument_info_.value_list_ = value_list; } void SetNames(std::vector&& names) { argument_info_.names_ = names; } void SetNames(std::initializer_list names) { argument_info_.names_ = names; } void SetHelp(std::optional&& val) { argument_info_.help_ = val; } void SetCategory(std::optional&& val) { argument_info_.category_ = val; } void SetMetavar(std::optional&& val) { argument_info_.metavar_ = val; } private: // Copying is bad. Move only. ArgumentBuilder(const ArgumentBuilder&) = delete; // Called by any function that doesn't chain back into this builder. // Completes the argument builder and save the information into the main builder. void CompleteArgument() { assert(save_value_specified_ && "No Into... function called, nowhere to save parsed values to"); assert(load_value_specified_ && "No Into... function called, nowhere to load parsed values from"); argument_info_.CompleteArgument(); // Appending the completed argument is destructive. The object is no longer // usable since all the useful information got moved out of it. AppendCompletedArgument(parent_, new detail::CmdlineParseArgument( std::move(argument_info_), std::move(save_value_), std::move(load_value_))); } friend struct CmdlineParser; friend struct CmdlineParser::Builder; friend struct CmdlineParser::UntypedArgumentBuilder; ArgumentBuilder(CmdlineParser::Builder& parser, std::shared_ptr save_destination) : parent_(parser), save_value_specified_(false), load_value_specified_(false), save_destination_(save_destination) { save_value_ = [](TArg&) { assert(false && "No save value function defined"); }; load_value_ = []() -> TArg& { assert(false && "No load value function defined"); __builtin_trap(); // Blow up. }; } CmdlineParser::Builder& parent_; std::function save_value_; std::function load_value_; bool save_value_specified_; bool load_value_specified_; detail::CmdlineParserArgumentInfo argument_info_; std::shared_ptr save_destination_; }; struct UntypedArgumentBuilder { // Set a type for this argument. The specific subcommand parser is looked up by the type. template ArgumentBuilder WithType() { return CreateTypedBuilder(); } UntypedArgumentBuilder& WithHelp(const char* sv) { SetHelp(sv); return *this; } UntypedArgumentBuilder& WithCategory(const char* sv) { SetCategory(sv); return *this; } UntypedArgumentBuilder& WithMetavar(const char* sv) { SetMetavar(sv); return *this; } // When used with multiple aliases, map the position of the alias to the value position. template ArgumentBuilder WithValues(std::initializer_list values) { auto&& a = CreateTypedBuilder(); a.WithValues(values); return std::move(a); } // When used with a single alias, map the alias into this value. // Same as 'WithValues({value})' , but allows the omission of the curly braces {}. template ArgumentBuilder WithValue(const TArg& value) { return WithValues({ value }); } // Set the current building argument to target this key. // When this command line argument is parsed, it can be fetched with this key. Builder& IntoKey(const TVariantMapKey& key) { return CreateTypedBuilder().IntoKey(key); } // Ensure we always move this when returning a new builder. UntypedArgumentBuilder(UntypedArgumentBuilder&&) noexcept = default; protected: void SetNames(std::vector&& names) { names_ = std::move(names); } void SetNames(std::initializer_list names) { names_ = names; } void SetHelp(std::optional sv) { help_.swap(sv); } void SetMetavar(std::optional sv) { metavar_.swap(sv); } void SetCategory(std::optional sv) { category_.swap(sv); } private: // No copying. Move instead. UntypedArgumentBuilder(const UntypedArgumentBuilder&) = delete; template ArgumentBuilder CreateTypedBuilder() { auto&& b = CreateArgumentBuilder(parent_); InitializeTypedBuilder(&b); // Type-specific initialization b.SetNames(std::move(names_)); b.SetHelp(std::move(help_)); b.SetCategory(std::move(category_)); b.SetMetavar(std::move(metavar_)); return std::move(b); } template typename std::enable_if::value>::type InitializeTypedBuilder(ArgumentBuilder* arg_builder) { // Every Unit argument implicitly maps to a runtime value of Unit{} std::vector values(names_.size(), Unit{}); arg_builder->SetValuesInternal(std::move(values)); } // No extra work for all other types void InitializeTypedBuilder(void*) {} template friend struct ArgumentBuilder; friend struct Builder; explicit UntypedArgumentBuilder(CmdlineParser::Builder& parent) : parent_(parent) {} // UntypedArgumentBuilder(UntypedArgumentBuilder&& other) = default; CmdlineParser::Builder& parent_; std::vector names_; std::optional category_; std::optional help_; std::optional metavar_; }; // Build a new parser given a chain of calls to define arguments. struct Builder { Builder() : save_destination_(new SaveDestination()) {} // Define a single argument. The default type is Unit. UntypedArgumentBuilder Define(const char* name) { return Define({name}); } Builder& ClearCategory() { default_category_.reset(); return *this; } Builder& SetCategory(const char* sv) { default_category_ = sv; return *this; } Builder& OrderCategories(std::vector categories) { category_order_.swap(categories); return *this; } // Define a single argument with multiple aliases. UntypedArgumentBuilder Define(std::initializer_list names) { auto&& b = UntypedArgumentBuilder(*this); b.SetNames(names); b.SetCategory(default_category_); return std::move(b); } // Whether the parser should give up on unrecognized arguments. Not recommended. Builder& IgnoreUnrecognized(bool ignore_unrecognized) { ignore_unrecognized_ = ignore_unrecognized; return *this; } // Provide a list of arguments to ignore for backwards compatibility. Builder& Ignore(std::initializer_list ignore_list) { auto current_cat = default_category_; default_category_ = "Ignored"; for (auto&& ignore_name : ignore_list) { std::string ign = ignore_name; // Ignored arguments are just like a regular definition which have very // liberal parsing requirements (no range checks, no value checks). // Unlike regular argument definitions, when a value gets parsed into its // stronger type, we just throw it away. if (ign.find('_') != std::string::npos) { // Does the arg-def have a wildcard? // pretend this is a string, e.g. -Xjitconfig: auto&& builder = Define(ignore_name).template WithType().IntoIgnore(); assert(&builder == this); (void)builder; // Ignore pointless unused warning, it's used in the assert. } else { // pretend this is a unit, e.g. -Xjitblocking auto&& builder = Define(ignore_name).template WithType().IntoIgnore(); assert(&builder == this); (void)builder; // Ignore pointless unused warning, it's used in the assert. } } ignore_list_ = ignore_list; default_category_ = current_cat; return *this; } // Finish building the parser; performs a check of the validity. Return value is moved, not // copied. Do not call this more than once. CmdlineParser Build() { assert(!built_); built_ = true; auto&& p = CmdlineParser(ignore_unrecognized_, std::move(ignore_list_), save_destination_, std::move(completed_arguments_), std::move(category_order_)); return std::move(p); } protected: void AppendCompletedArgument(detail::CmdlineParseArgumentAny* arg) { auto smart_ptr = std::unique_ptr(arg); completed_arguments_.push_back(std::move(smart_ptr)); } private: // No copying now! Builder(const Builder& other) = delete; template friend struct ArgumentBuilder; friend struct UntypedArgumentBuilder; friend struct CmdlineParser; bool built_ = false; bool ignore_unrecognized_ = false; std::vector ignore_list_; std::shared_ptr save_destination_; std::optional default_category_; std::vector category_order_; std::vector> completed_arguments_; }; void DumpHelp(VariableIndentationOutputStream& vios); CmdlineResult Parse(const std::string& argv) { std::vector tokenized; Split(argv, ' ', &tokenized); return Parse(TokenRange(std::move(tokenized))); } // Parse the arguments; storing results into the arguments map. Returns success value. CmdlineResult Parse(const char* argv) { return Parse(std::string(argv)); } // Parse the arguments; storing the results into the arguments map. Returns success value. // Assumes that argv[0] is a valid argument (i.e. not the program name). CmdlineResult Parse(const std::vector& argv) { return Parse(TokenRange(argv.begin(), argv.end())); } // Parse the arguments; storing the results into the arguments map. Returns success value. // Assumes that argv[0] is a valid argument (i.e. not the program name). CmdlineResult Parse(const std::vector& argv) { return Parse(TokenRange(argv.begin(), argv.end())); } // Parse the arguments (directly from an int main(argv,argc)). Returns success value. // Assumes that argv[0] is the program name, and ignores it. CmdlineResult Parse(const char* argv[], int argc) { return Parse(TokenRange(&argv[1], argc - 1)); // ignore argv[0] because it's the program name } // Look up the arguments that have been parsed; use the target keys to lookup individual args. const TVariantMap& GetArgumentsMap() const { return save_destination_->GetMap(); } // Release the arguments map that has been parsed; useful for move semantics. TVariantMap&& ReleaseArgumentsMap() { return save_destination_->ReleaseMap(); } // How many arguments were defined? size_t CountDefinedArguments() const { return completed_arguments_.size(); } // Ensure we have a default move constructor. CmdlineParser(CmdlineParser&&) noexcept = default; // Ensure we have a default move assignment operator. CmdlineParser& operator=(CmdlineParser&&) noexcept = default; private: friend struct Builder; // Construct a new parser from the builder. Move all the arguments. CmdlineParser(bool ignore_unrecognized, std::vector&& ignore_list, std::shared_ptr save_destination, std::vector>&& completed_arguments, std::vector&& category_order) : ignore_unrecognized_(ignore_unrecognized), ignore_list_(std::move(ignore_list)), save_destination_(save_destination), completed_arguments_(std::move(completed_arguments)), category_order_(category_order) { assert(save_destination != nullptr); } // Parse the arguments; storing results into the arguments map. Returns success value. // The parsing will fail on the first non-success parse result and return that error. // // All previously-parsed arguments are cleared out. // Otherwise, all parsed arguments will be stored into SaveDestination as a side-effect. // A partial parse will result only in a partial save of the arguments. CmdlineResult Parse(TokenRange&& arguments_list) { save_destination_->Clear(); for (size_t i = 0; i < arguments_list.Size(); ) { TokenRange possible_name = arguments_list.Slice(i); size_t best_match_size = 0; // How many tokens were matched in the best case. size_t best_match_arg_idx = 0; bool matched = false; // At least one argument definition has been matched? // Find the closest argument definition for the remaining token range. size_t arg_idx = 0; for (auto&& arg : completed_arguments_) { size_t local_match = arg->MaybeMatches(possible_name); if (local_match > best_match_size) { best_match_size = local_match; best_match_arg_idx = arg_idx; matched = true; } arg_idx++; } // Saw some kind of unknown argument if (matched == false) { if (UNLIKELY(ignore_unrecognized_)) { // This is usually off, we only need it for JNI. // Consume 1 token and keep going, hopefully the next token is a good one. ++i; continue; } // Common case: // Bail out on the first unknown argument with an error. return CmdlineResult(CmdlineResult::kUnknown, std::string("Unknown argument: ") + possible_name[0]); } // Look at the best-matched argument definition and try to parse against that. auto&& arg = completed_arguments_[best_match_arg_idx]; assert(arg->MaybeMatches(possible_name) == best_match_size); // Try to parse the argument now, if we have enough tokens. std::pair num_tokens = arg->GetNumTokens(); size_t min_tokens; size_t max_tokens; std::tie(min_tokens, max_tokens) = num_tokens; if ((i + min_tokens) > arguments_list.Size()) { // expected longer command line but it was too short // e.g. if the argv was only "-Xms" without specifying a memory option CMDLINE_DEBUG_LOG << "Parse failure, i = " << i << ", arg list " << arguments_list.Size() << " num tokens in arg_def: " << min_tokens << "," << max_tokens << std::endl; return CmdlineResult(CmdlineResult::kFailure, std::string("Argument ") + possible_name[0] + ": incomplete command line arguments, expected " + std::to_string(size_t(i + min_tokens) - arguments_list.Size()) + " more tokens"); } if (best_match_size > max_tokens || best_match_size < min_tokens) { // Even our best match was out of range, so parsing would fail instantly. return CmdlineResult(CmdlineResult::kFailure, std::string("Argument ") + possible_name[0] + ": too few tokens " "matched " + std::to_string(best_match_size) + " but wanted " + std::to_string(num_tokens.first)); } // We have enough tokens to begin exact parsing. TokenRange exact_range = possible_name.Slice(0, max_tokens); size_t consumed_tokens = 1; // At least 1 if we ever want to try to resume parsing on error CmdlineResult parse_attempt = arg->ParseArgument(exact_range, &consumed_tokens); if (parse_attempt.IsError()) { // We may also want to continue parsing the other tokens to gather more errors. return parse_attempt; } // else the value has been successfully stored into the map assert(consumed_tokens > 0); // Don't hang in an infinite loop trying to parse i += consumed_tokens; // TODO: also handle ignoring arguments for backwards compatibility } // for return CmdlineResult(CmdlineResult::kSuccess); } bool ignore_unrecognized_ = false; std::vector ignore_list_; std::shared_ptr save_destination_; std::vector> completed_arguments_; std::vector category_order_; }; // This has to be defined after everything else, since we want the builders to call this. template class TVariantMapKey> template typename CmdlineParser::template ArgumentBuilder CmdlineParser::CreateArgumentBuilder( CmdlineParser::Builder& parent) { return CmdlineParser::ArgumentBuilder( parent, parent.save_destination_); } // This has to be defined after everything else, since we want the builders to call this. template class TVariantMapKey> void CmdlineParser::AppendCompletedArgument( CmdlineParser::Builder& builder, detail::CmdlineParseArgumentAny* arg) { builder.AppendCompletedArgument(arg); } template class TVariantMapKey> void CmdlineParser::DumpHelp(VariableIndentationOutputStream& vios) { std::vector uncat; std::unordered_map> args; for (const std::unique_ptr& it : completed_arguments_) { auto cat = it->GetCategory(); if (cat.has_value()) { if (args.find(*cat) == args.end()) { args[*cat] = {}; } args.at(*cat).push_back(it.get()); } else { uncat.push_back(it.get()); } } args.erase("Ignored"); for (auto arg : uncat) { arg->DumpHelp(vios); vios.Stream(); } for (auto it : category_order_) { auto cur = args.find(it); if (cur != args.end() && !cur->second.empty()) { vios.Stream() << "The following " << it << " arguments are supported:" << std::endl; ScopedIndentation si(&vios); for (detail::CmdlineParseArgumentAny* arg : cur->second) { arg->DumpHelp(vios); vios.Stream(); } args.erase(cur->first); } } for (auto [cat, lst] : args) { vios.Stream() << "The following " << cat << " arguments are supported:" << std::endl; ScopedIndentation si(&vios); for (auto& arg : completed_arguments_) { arg->DumpHelp(vios); vios.Stream(); } } if (!ignore_list_.empty()) { vios.Stream() << "The following arguments are ignored for compatibility:" << std::endl; ScopedIndentation si(&vios); for (auto ign : ignore_list_) { vios.Stream() << ign << std::endl; } } } } // namespace art #endif // ART_CMDLINE_CMDLINE_PARSER_H_