diff --git a/js/src/irregexp/imported/gen-regexp-special-case.cc b/js/src/irregexp/imported/gen-regexp-special-case.cc --- a/js/src/irregexp/imported/gen-regexp-special-case.cc +++ b/js/src/irregexp/imported/gen-regexp-special-case.cc @@ -6,10 +6,11 @@ #include #include #include #include "irregexp/imported/special-case.h" +#include "unicode/usetiter.h" namespace v8 { namespace internal { static const base::uc32 kSurrogateStart = 0xd800; @@ -123,10 +124,56 @@ PrintSet(out, "IgnoreSet", ignore); PrintSet(out, "SpecialAddSet", special_add); } +void PrintUnicodeSpecial(std::ofstream& out) { + icu::UnicodeSet non_simple_folding; + icu::UnicodeSet current; + UErrorCode status = U_ZERO_ERROR; + // Look at all characters except white spaces. + icu::UnicodeSet interestingCP(u"[^[:White_Space:]]", status); + CHECK_EQ(status, U_ZERO_ERROR); + icu::UnicodeSetIterator iter(interestingCP); + while (iter.next()) { + UChar32 c = iter.getCodepoint(); + current.set(c, c); + current.closeOver(USET_CASE_INSENSITIVE).removeAllStrings(); + CHECK(!current.isBogus()); + // Remove characters from the closeover that have a simple case folding. + icu::UnicodeSet toRemove; + icu::UnicodeSetIterator closeOverIter(current); + while (closeOverIter.next()) { + UChar32 closeOverChar = closeOverIter.getCodepoint(); + UChar32 closeOverSCF = u_foldCase(closeOverChar, U_FOLD_CASE_DEFAULT); + if (closeOverChar != closeOverSCF) { + toRemove.add(closeOverChar); + } + } + CHECK(!toRemove.isBogus()); + current.removeAll(toRemove); + + // The current character and its simple case folding are also always OK. + UChar32 scf = u_foldCase(c, U_FOLD_CASE_DEFAULT); + current.remove(c); + current.remove(scf); + + // If there are any characters remaining, they were added due to full case + // foldings and shouldn't match the current charcter according to the spec. + if (!current.isEmpty()) { + // Ensure that the character doesn't have a simple case folding. + // Otherwise the current approach of simply removing the character from + // the set before calling closeOver won't work. + CHECK_EQ(c, scf); + non_simple_folding.add(c); + } + } + CHECK(!non_simple_folding.isBogus()); + + PrintSet(out, "UnicodeNonSimpleCloseOverSet", non_simple_folding); +} + void WriteHeader(const char* header_filename) { std::ofstream out(header_filename); out << std::hex << std::setfill('0') << std::setw(4); out << "// Copyright 2020 the V8 project authors. All rights reserved.\n" << "// Use of this source code is governed by a BSD-style license that\n" @@ -143,10 +190,11 @@ << "#include \"unicode/uniset.h\"\n" << "namespace v8 {\n" << "namespace internal {\n\n"; PrintSpecial(out); + PrintUnicodeSpecial(out); out << "\n" << "} // namespace internal\n" << "} // namespace v8\n" << "#endif // V8_INTL_SUPPORT\n"; diff --git a/js/src/irregexp/imported/regexp-ast.h b/js/src/irregexp/imported/regexp-ast.h --- a/js/src/irregexp/imported/regexp-ast.h +++ b/js/src/irregexp/imported/regexp-ast.h @@ -3,20 +3,25 @@ // found in the LICENSE file. #ifndef V8_REGEXP_REGEXP_AST_H_ #define V8_REGEXP_REGEXP_AST_H_ +#ifdef V8_INTL_SUPPORT +#include "unicode/uniset.h" +#endif // V8_INTL_SUPPORT #include "irregexp/RegExpShim.h" namespace v8 { namespace internal { #define FOR_EACH_REG_EXP_TREE_TYPE(VISIT) \ VISIT(Disjunction) \ VISIT(Alternative) \ VISIT(Assertion) \ - VISIT(CharacterClass) \ + VISIT(ClassRanges) \ + VISIT(ClassSetOperand) \ + VISIT(ClassSetExpression) \ VISIT(Atom) \ VISIT(Quantifier) \ VISIT(Capture) \ VISIT(Group) \ VISIT(Lookaround) \ @@ -111,43 +116,76 @@ // Add class escapes. Add case equivalent closure for \w and \W if necessary. V8_EXPORT_PRIVATE static void AddClassEscape( StandardCharacterSet standard_character_set, ZoneList* ranges, bool add_unicode_case_equivalents, Zone* zone); + // Add case equivalents to ranges. Only used for /i, not for /ui or /vi, as + // the semantics for unicode mode are slightly different. + // See https://tc39.es/ecma262/#sec-runtime-semantics-canonicalize-ch Note 4. V8_EXPORT_PRIVATE static void AddCaseEquivalents( Isolate* isolate, Zone* zone, ZoneList* ranges, bool is_one_byte); + // Add case equivalent code points to ranges. Only used for /ui and /vi, not + // for /i, as the semantics for non-unicode mode are slightly different. + // See https://tc39.es/ecma262/#sec-runtime-semantics-canonicalize-ch Note 4. + static void AddUnicodeCaseEquivalents(ZoneList* ranges, + Zone* zone); + +#ifdef V8_INTL_SUPPORT + // Creates the closeOver of the given UnicodeSet, removing all + // characters/strings that can't be derived via simple case folding. + static void UnicodeSimpleCloseOver(icu::UnicodeSet& set); +#endif // V8_INTL_SUPPORT bool Contains(base::uc32 i) const { return from_ <= i && i <= to_; } base::uc32 from() const { return from_; } base::uc32 to() const { return to_; } bool IsEverything(base::uc32 max) const { return from_ == 0 && to_ >= max; } bool IsSingleton() const { return from_ == to_; } + // Whether a range list is in canonical form: Ranges ordered by from value, // and ranges non-overlapping and non-adjacent. - V8_EXPORT_PRIVATE static bool IsCanonical(ZoneList* ranges); + V8_EXPORT_PRIVATE static bool IsCanonical( + const ZoneList* ranges); // Convert range list to canonical form. The characters covered by the ranges // will still be the same, but no character is in more than one range, and // adjacent ranges are merged. The resulting list may be shorter than the // original, but cannot be longer. static void Canonicalize(ZoneList* ranges); // Negate the contents of a character range in canonical form. - static void Negate(ZoneList* src, + static void Negate(const ZoneList* src, ZoneList* dst, Zone* zone); - + // Intersect the contents of two character ranges in canonical form. + static void Intersect(const ZoneList* lhs, + const ZoneList* rhs, + ZoneList* dst, Zone* zone); + // Subtract the contents of |to_remove| from the contents of |src|. + static void Subtract(const ZoneList* src, + const ZoneList* to_remove, + ZoneList* dst, Zone* zone); // Remove all ranges outside the one-byte range. static void ClampToOneByte(ZoneList* ranges); + // Checks if two ranges (both need to be canonical) are equal. + static bool Equals(const ZoneList* lhs, + const ZoneList* rhs); private: CharacterRange(base::uc32 from, base::uc32 to) : from_(from), to_(to) {} static constexpr int kMaxCodePoint = 0x10ffff; base::uc32 from_ = 0; base::uc32 to_ = 0; }; +inline bool operator==(const CharacterRange& lhs, const CharacterRange& rhs) { + return lhs.from() == rhs.from() && lhs.to() == rhs.to(); +} +inline bool operator!=(const CharacterRange& lhs, const CharacterRange& rhs) { + return !operator==(lhs, rhs); +} + #define DECL_BOILERPLATE(Name) \ void* Accept(RegExpVisitor* visitor, void* data) override; \ RegExpNode* ToNode(RegExpCompiler* compiler, RegExpNode* on_success) \ override; \ RegExp##Name* As##Name() override; \ @@ -265,36 +303,35 @@ private: ZoneList* ranges_ = nullptr; base::Optional standard_set_type_; }; -class RegExpCharacterClass final : public RegExpTree { +class RegExpClassRanges final : public RegExpTree { public: // NEGATED: The character class is negated and should match everything but // the specified ranges. // CONTAINS_SPLIT_SURROGATE: The character class contains part of a split // surrogate and should not be unicode-desugared (crbug.com/641091). enum Flag { NEGATED = 1 << 0, CONTAINS_SPLIT_SURROGATE = 1 << 1, }; - using CharacterClassFlags = base::Flags; + using ClassRangesFlags = base::Flags; - RegExpCharacterClass( - Zone* zone, ZoneList* ranges, - CharacterClassFlags character_class_flags = CharacterClassFlags()) - : set_(ranges), character_class_flags_(character_class_flags) { + RegExpClassRanges(Zone* zone, ZoneList* ranges, + ClassRangesFlags class_ranges_flags = ClassRangesFlags()) + : set_(ranges), class_ranges_flags_(class_ranges_flags) { // Convert the empty set of ranges to the negated Everything() range. if (ranges->is_empty()) { ranges->Add(CharacterRange::Everything(), zone); - character_class_flags_ ^= NEGATED; + class_ranges_flags_ ^= NEGATED; } } - explicit RegExpCharacterClass(StandardCharacterSet standard_set_type) - : set_(standard_set_type), character_class_flags_() {} + explicit RegExpClassRanges(StandardCharacterSet standard_set_type) + : set_(standard_set_type), class_ranges_flags_() {} - DECL_BOILERPLATE(CharacterClass); + DECL_BOILERPLATE(ClassRanges); bool IsTextElement() override { return true; } int min_match() override { return 1; } // The character class may match two code units for unicode regexps. // TODO(yangguo): we should split this class for usage in TextElement, and @@ -313,18 +350,115 @@ } CharacterSet character_set() const { return set_; } ZoneList* ranges(Zone* zone) { return set_.ranges(zone); } - bool is_negated() const { return (character_class_flags_ & NEGATED) != 0; } + bool is_negated() const { return (class_ranges_flags_ & NEGATED) != 0; } bool contains_split_surrogate() const { - return (character_class_flags_ & CONTAINS_SPLIT_SURROGATE) != 0; + return (class_ranges_flags_ & CONTAINS_SPLIT_SURROGATE) != 0; } private: CharacterSet set_; - CharacterClassFlags character_class_flags_; + ClassRangesFlags class_ranges_flags_; +}; + +struct CharacterClassStringLess { + bool operator()(const base::Vector& lhs, + const base::Vector& rhs) const { + // Longer strings first so we generate matches for the largest string + // possible. + if (lhs.length() != rhs.length()) { + return lhs.length() > rhs.length(); + } + for (int i = 0; i < lhs.length(); i++) { + if (lhs[i] != rhs[i]) { + return lhs[i] < rhs[i]; + } + } + return false; + } +}; + +// A type used for strings as part of character classes (only possible in +// unicode sets mode). +// We use a ZoneMap instead of an UnorderedZoneMap because we need to match +// the longest alternatives first. By using a ZoneMap with the custom comparator +// we can avoid sorting before assembling the code. +// Strings are likely short (the largest string in current unicode properties +// consists of 10 code points). +using CharacterClassStrings = ZoneMap, + RegExpTree*, CharacterClassStringLess>; + +// TODO(pthier): If we are sure we don't want to use icu::UnicodeSets +// (performance evaluation pending), this class can be merged with +// RegExpClassRanges. +class RegExpClassSetOperand final : public RegExpTree { + public: + RegExpClassSetOperand(ZoneList* ranges, + CharacterClassStrings* strings); + + DECL_BOILERPLATE(ClassSetOperand); + + bool IsTextElement() override { return true; } + int min_match() override { return min_match_; } + int max_match() override { return max_match_; } + + void Union(RegExpClassSetOperand* other, Zone* zone); + void Intersect(RegExpClassSetOperand* other, + ZoneList* temp_ranges, Zone* zone); + void Subtract(RegExpClassSetOperand* other, + ZoneList* temp_ranges, Zone* zone); + + bool has_strings() const { return !strings_->empty(); } + ZoneList* ranges() { return ranges_; } + CharacterClassStrings* strings() { return strings_; } + + private: + ZoneList* ranges_; + CharacterClassStrings* strings_; + int min_match_; + int max_match_; +}; + +class RegExpClassSetExpression final : public RegExpTree { + public: + enum class OperationType { kUnion, kIntersection, kSubtraction }; + + RegExpClassSetExpression(OperationType op, bool is_negated, + bool may_contain_strings, + ZoneList* operands); + + DECL_BOILERPLATE(ClassSetExpression); + + bool IsTextElement() override { return true; } + int min_match() override { return 0; } + int max_match() override { return max_match_; } + + OperationType operation() const { return operation_; } + bool is_negated() const { return is_negated_; } + bool may_contain_strings() const { return may_contain_strings_; } + const ZoneList* operands() const { return operands_; } + ZoneList* operands() { return operands_; } + + private: + // Recursively evaluates the tree rooted at |root|, computing the valid + // CharacterRanges and strings after applying all set operations. + // The original tree will be modified by this method, so don't store pointers + // to inner nodes of the tree somewhere else! + // Modifying the tree in-place saves memory and speeds up multiple calls of + // the method (e.g. when unrolling quantifiers). + // |temp_ranges| is used for intermediate results, passed as parameter to + // avoid allocating new lists all the time. + static RegExpClassSetOperand* ComputeExpression( + RegExpTree* root, ZoneList* temp_ranges, Zone* zone); + + const OperationType operation_; + const bool is_negated_; + const bool may_contain_strings_; + ZoneList* operands_ = nullptr; + int max_match_; }; class RegExpAtom final : public RegExpTree { public: explicit RegExpAtom(base::Vector data) : data_(data) {} @@ -343,14 +477,14 @@ base::Vector data_; }; class TextElement final { public: - enum TextType { ATOM, CHAR_CLASS }; + enum TextType { ATOM, CLASS_RANGES }; static TextElement Atom(RegExpAtom* atom); - static TextElement CharClass(RegExpCharacterClass* char_class); + static TextElement ClassRanges(RegExpClassRanges* class_ranges); int cp_offset() const { return cp_offset_; } void set_cp_offset(int cp_offset) { cp_offset_ = cp_offset; } int length() const; @@ -361,13 +495,13 @@ RegExpAtom* atom() const { DCHECK(text_type() == ATOM); return reinterpret_cast(tree()); } - RegExpCharacterClass* char_class() const { - DCHECK(text_type() == CHAR_CLASS); - return reinterpret_cast(tree()); + RegExpClassRanges* class_ranges() const { + DCHECK(text_type() == CLASS_RANGES); + return reinterpret_cast(tree()); } private: TextElement(TextType text_type, RegExpTree* tree) : cp_offset_(-1), text_type_(text_type), tree_(tree) {} diff --git a/js/src/irregexp/imported/regexp-ast.cc b/js/src/irregexp/imported/regexp-ast.cc --- a/js/src/irregexp/imported/regexp-ast.cc +++ b/js/src/irregexp/imported/regexp-ast.cc @@ -179,23 +179,59 @@ if (!that.IsSingleton()) { os_ << "-" << AsUC32(that.to()); } } - -void* RegExpUnparser::VisitCharacterClass(RegExpCharacterClass* that, - void* data) { +void* RegExpUnparser::VisitClassRanges(RegExpClassRanges* that, void* data) { if (that->is_negated()) os_ << "^"; os_ << "["; for (int i = 0; i < that->ranges(zone_)->length(); i++) { if (i > 0) os_ << " "; VisitCharacterRange(that->ranges(zone_)->at(i)); } os_ << "]"; return nullptr; } +void* RegExpUnparser::VisitClassSetOperand(RegExpClassSetOperand* that, + void* data) { + os_ << "!["; + for (int i = 0; i < that->ranges()->length(); i++) { + if (i > 0) os_ << " "; + VisitCharacterRange(that->ranges()->at(i)); + } + for (auto iter : *that->strings()) { + os_ << " '"; + os_ << std::string(iter.first.begin(), iter.first.end()); + os_ << "'"; + } + os_ << "]"; + return nullptr; +} + +void* RegExpUnparser::VisitClassSetExpression(RegExpClassSetExpression* that, + void* data) { + switch (that->operation()) { + case RegExpClassSetExpression::OperationType::kUnion: + os_ << "++"; + break; + case RegExpClassSetExpression::OperationType::kIntersection: + os_ << "&&"; + break; + case RegExpClassSetExpression::OperationType::kSubtraction: + os_ << "--"; + break; + } + if (that->is_negated()) os_ << "^"; + os_ << "["; + for (int i = 0; i < that->operands()->length(); i++) { + if (i > 0) os_ << " "; + that->operands()->at(i)->Accept(this, data); + } + os_ << "]"; + return nullptr; +} void* RegExpUnparser::VisitAssertion(RegExpAssertion* that, void* data) { switch (that->assertion_type()) { case RegExpAssertion::Type::START_OF_INPUT: os_ << "@^i"; @@ -339,8 +375,39 @@ int node_max_match = node->max_match(); max_match_ = IncreaseBy(max_match_, node_max_match); } } +RegExpClassSetOperand::RegExpClassSetOperand(ZoneList* ranges, + CharacterClassStrings* strings) + : ranges_(ranges), strings_(strings) { + DCHECK_NOT_NULL(ranges); + DCHECK_NOT_NULL(strings); + min_match_ = 0; + max_match_ = 0; + if (!ranges->is_empty()) { + min_match_ = 1; + max_match_ = 2; + } + for (auto string : *strings) { + min_match_ = std::min(min_match_, string.second->min_match()); + max_match_ = std::max(max_match_, string.second->max_match()); + } +} + +RegExpClassSetExpression::RegExpClassSetExpression( + OperationType op, bool is_negated, bool may_contain_strings, + ZoneList* operands) + : operation_(op), + is_negated_(is_negated), + may_contain_strings_(may_contain_strings), + operands_(operands) { + DCHECK_NOT_NULL(operands); + DCHECK_IMPLIES(is_negated_, !may_contain_strings_); + max_match_ = 0; + for (auto op : *operands) { + max_match_ = std::max(max_match_, op->max_match()); + } +} } // namespace internal } // namespace v8 diff --git a/js/src/irregexp/imported/regexp-bytecode-generator.cc b/js/src/irregexp/imported/regexp-bytecode-generator.cc --- a/js/src/irregexp/imported/regexp-bytecode-generator.cc +++ b/js/src/irregexp/imported/regexp-bytecode-generator.cc @@ -376,11 +376,11 @@ Handle RegExpBytecodeGenerator::GetCode(Handle source) { Bind(&backtrack_); Backtrack(); Handle array; - if (FLAG_regexp_peephole_optimization) { + if (v8_flags.regexp_peephole_optimization) { array = RegExpBytecodePeepholeOptimization::OptimizeBytecode( isolate_, zone(), source, buffer_.data(), length(), jump_edges_); } else { array = isolate_->factory()->NewByteArray(length()); Copy(array->GetDataStartAddress()); diff --git a/js/src/irregexp/imported/regexp-bytecode-peephole.cc b/js/src/irregexp/imported/regexp-bytecode-peephole.cc --- a/js/src/irregexp/imported/regexp-bytecode-peephole.cc +++ b/js/src/irregexp/imported/regexp-bytecode-peephole.cc @@ -1010,11 +1010,11 @@ RegExpBytecodePeephole peephole(zone, length, jump_edges); bool did_optimize = peephole.OptimizeBytecode(bytecode, length); Handle array = isolate->factory()->NewByteArray(peephole.Length()); peephole.CopyOptimizedBytecode(array->GetDataStartAddress()); - if (did_optimize && FLAG_trace_regexp_peephole_optimization) { + if (did_optimize && v8_flags.trace_regexp_peephole_optimization) { PrintF("Original Bytecode:\n"); RegExpBytecodeDisassemble(bytecode, length, source->ToCString().get()); PrintF("Optimized Bytecode:\n"); RegExpBytecodeDisassemble(array->GetDataStartAddress(), peephole.Length(), source->ToCString().get()); diff --git a/js/src/irregexp/imported/regexp-compiler-tonode.cc b/js/src/irregexp/imported/regexp-compiler-tonode.cc --- a/js/src/irregexp/imported/regexp-compiler-tonode.cc +++ b/js/src/irregexp/imported/regexp-compiler-tonode.cc @@ -87,11 +87,11 @@ return true; } } // namespace -bool RegExpCharacterClass::is_standard(Zone* zone) { +bool RegExpClassRanges::is_standard(Zone* zone) { // TODO(lrn): Remove need for this function, by not throwing away information // along the way. if (is_negated()) { return false; } @@ -413,13 +413,38 @@ ZoneList* range = CharacterRange::List(zone, CharacterRange::Range(0, kMaxUtf16CodeUnit)); return TextNode::CreateForCharacterRanges(zone, range, false, on_success); } -void AddUnicodeCaseEquivalents(ZoneList* ranges, Zone* zone) { +} // namespace + #ifdef V8_INTL_SUPPORT - DCHECK(CharacterRange::IsCanonical(ranges)); +// static +void CharacterRange::UnicodeSimpleCloseOver(icu::UnicodeSet& set) { + // Remove characters for which closeOver() adds full-case-folding equivalents + // because we should work only with simple case folding mappings. + icu::UnicodeSet non_simple = icu::UnicodeSet(set); + non_simple.retainAll(RegExpCaseFolding::UnicodeNonSimpleCloseOverSet()); + set.removeAll(non_simple); + + set.closeOver(USET_CASE_INSENSITIVE); + // Full case folding maps single characters to multiple characters. + // Those are represented as strings in the set. Remove them so that + // we end up with only simple and common case mappings. + set.removeAllStrings(); + + // Add characters that have non-simple case foldings again (they match + // themselves). + set.addAll(non_simple); +} +#endif // V8_INTL_SUPPORT + +// static +void CharacterRange::AddUnicodeCaseEquivalents(ZoneList* ranges, + Zone* zone) { +#ifdef V8_INTL_SUPPORT + DCHECK(IsCanonical(ranges)); // Micro-optimization to avoid passing large ranges to UnicodeSet::closeOver. // See also https://crbug.com/v8/6727. // TODO(jgruber): This only covers the special case of the {0,0x10FFFF} range, // which we use frequently internally. But large ranges can also easily be @@ -432,51 +457,56 @@ for (int i = 0; i < ranges->length(); i++) { set.add(ranges->at(i).from(), ranges->at(i).to()); } // Clear the ranges list without freeing the backing store. ranges->Rewind(0); - set.closeOver(USET_CASE_INSENSITIVE); - // Full case mapping map single characters to multiple characters. - // Those are represented as strings in the set. Remove them so that - // we end up with only simple and common case mappings. - set.removeAllStrings(); + + UnicodeSimpleCloseOver(set); for (int i = 0; i < set.getRangeCount(); i++) { - ranges->Add(CharacterRange::Range(set.getRangeStart(i), set.getRangeEnd(i)), - zone); + ranges->Add(Range(set.getRangeStart(i), set.getRangeEnd(i)), zone); } // No errors and everything we collected have been ranges. - CharacterRange::Canonicalize(ranges); + Canonicalize(ranges); #endif // V8_INTL_SUPPORT } -} // namespace - -RegExpNode* RegExpCharacterClass::ToNode(RegExpCompiler* compiler, - RegExpNode* on_success) { +RegExpNode* RegExpClassRanges::ToNode(RegExpCompiler* compiler, + RegExpNode* on_success) { set_.Canonicalize(); Zone* const zone = compiler->zone(); ZoneList* ranges = this->ranges(zone); if (NeedsUnicodeCaseEquivalents(compiler->flags())) { - AddUnicodeCaseEquivalents(ranges, zone); + CharacterRange::AddUnicodeCaseEquivalents(ranges, zone); } - if (!IsUnicode(compiler->flags()) || compiler->one_byte() || + if (!IsEitherUnicode(compiler->flags()) || compiler->one_byte() || contains_split_surrogate()) { return zone->New(this, compiler->read_backward(), on_success); } if (is_negated()) { + // With /v, character classes are never negated. + // TODO(v8:11935): Change permalink once proposal is in stage 4. + // https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#sec-compileatom + // Atom :: CharacterClass + // 4. Assert: cc.[[Invert]] is false. + // Instead the complement is created when evaluating the class set. + // The only exception is the "nothing range" (negated everything), which is + // internally created for an empty set. + DCHECK_IMPLIES( + IsUnicodeSets(compiler->flags()), + ranges->length() == 1 && ranges->first().IsEverything(kMaxCodePoint)); ZoneList* negated = zone->New>(2, zone); CharacterRange::Negate(ranges, negated, zone); ranges = negated; } if (ranges->length() == 0) { // The empty character class is used as a 'fail' node. - RegExpCharacterClass* fail = zone->New(zone, ranges); + RegExpClassRanges* fail = zone->New(zone, ranges); return zone->New(fail, compiler->read_backward(), on_success); } if (set_.is_standard() && standard_type() == StandardCharacterSet::kEverything) { @@ -499,10 +529,151 @@ if (ranges->length() > kMaxRangesToInline) result->SetDoNotInline(); return result; } +RegExpNode* RegExpClassSetOperand::ToNode(RegExpCompiler* compiler, + RegExpNode* on_success) { + Zone* zone = compiler->zone(); + const int size = (has_strings() ? static_cast(strings()->size()) : 0) + + (ranges()->is_empty() ? 0 : 1); + if (size == 0) { + // If neither ranges nor strings are present, the operand is equal to an + // empty range (matching nothing). + ZoneList* empty = + zone->template New>(0, zone); + return zone->template New(zone, empty) + ->ToNode(compiler, on_success); + } + ZoneList* alternatives = + zone->template New>(size, zone); + // Strings are sorted by length first (larger strings before shorter ones). + // See the comment on CharacterClassStrings. + // Empty strings (if present) are added after character ranges. + RegExpTree* empty_string = nullptr; + if (has_strings()) { + for (auto string : *strings()) { + if (string.second->IsEmpty()) { + empty_string = string.second; + } else { + alternatives->Add(string.second, zone); + } + } + } + if (!ranges()->is_empty()) { + alternatives->Add(zone->template New(zone, ranges()), + zone); + } + if (empty_string != nullptr) { + alternatives->Add(empty_string, zone); + } + + RegExpTree* node = nullptr; + if (size == 1) { + DCHECK_EQ(alternatives->length(), 1); + node = alternatives->first(); + } else { + node = zone->template New(alternatives); + } + return node->ToNode(compiler, on_success); +} + +RegExpNode* RegExpClassSetExpression::ToNode(RegExpCompiler* compiler, + RegExpNode* on_success) { + Zone* zone = compiler->zone(); + ZoneList* temp_ranges = + zone->template New>(4, zone); + RegExpClassSetOperand* root = ComputeExpression(this, temp_ranges, zone); + return root->ToNode(compiler, on_success); +} + +void RegExpClassSetOperand::Union(RegExpClassSetOperand* other, Zone* zone) { + ranges()->AddAll(*other->ranges(), zone); + strings()->insert(other->strings()->begin(), other->strings()->end()); +} + +void RegExpClassSetOperand::Intersect(RegExpClassSetOperand* other, + ZoneList* temp_ranges, + Zone* zone) { + CharacterRange::Intersect(ranges(), other->ranges(), temp_ranges, zone); + std::swap(*ranges(), *temp_ranges); + temp_ranges->Rewind(0); + for (auto iter = strings()->begin(); iter != strings()->end();) { + if (other->strings()->find(iter->first) == other->strings()->end()) { + iter = strings()->erase(iter); + } else { + iter++; + } + } +} + +void RegExpClassSetOperand::Subtract(RegExpClassSetOperand* other, + ZoneList* temp_ranges, + Zone* zone) { + CharacterRange::Subtract(ranges(), other->ranges(), temp_ranges, zone); + std::swap(*ranges(), *temp_ranges); + temp_ranges->Rewind(0); + for (auto iter = strings()->begin(); iter != strings()->end();) { + if (other->strings()->find(iter->first) != other->strings()->end()) { + iter = strings()->erase(iter); + } else { + iter++; + } + } +} + +// static +RegExpClassSetOperand* RegExpClassSetExpression::ComputeExpression( + RegExpTree* root, ZoneList* temp_ranges, Zone* zone) { + DCHECK(temp_ranges->is_empty()); + if (root->IsClassSetOperand()) { + return root->AsClassSetOperand(); + } + DCHECK(root->IsClassSetExpression()); + RegExpClassSetExpression* node = root->AsClassSetExpression(); + RegExpClassSetOperand* result = + ComputeExpression(node->operands()->at(0), temp_ranges, zone); + switch (node->operation()) { + case OperationType::kUnion: { + for (int i = 1; i < node->operands()->length(); i++) { + RegExpClassSetOperand* op = + ComputeExpression(node->operands()->at(i), temp_ranges, zone); + result->Union(op, zone); + } + CharacterRange::Canonicalize(result->ranges()); + break; + } + case OperationType::kIntersection: { + for (int i = 1; i < node->operands()->length(); i++) { + RegExpClassSetOperand* op = + ComputeExpression(node->operands()->at(i), temp_ranges, zone); + result->Intersect(op, temp_ranges, zone); + } + break; + } + case OperationType::kSubtraction: { + for (int i = 1; i < node->operands()->length(); i++) { + RegExpClassSetOperand* op = + ComputeExpression(node->operands()->at(i), temp_ranges, zone); + result->Subtract(op, temp_ranges, zone); + } + break; + } + } + if (node->is_negated()) { + DCHECK(!result->has_strings()); + CharacterRange::Negate(result->ranges(), temp_ranges, zone); + std::swap(*result->ranges(), *temp_ranges); + temp_ranges->Rewind(0); + } + // Store the result as single operand of the current node. + node->operands()->Set(0, result); + node->operands()->Rewind(1); + + return result; +} + namespace { int CompareFirstChar(RegExpTree* const* a, RegExpTree* const* b) { RegExpAtom* atom1 = (*a)->AsAtom(); RegExpAtom* atom2 = (*b)->AsAtom(); @@ -764,11 +935,11 @@ alternatives->at(write_posn++) = alternatives->at(i); i++; continue; } const RegExpFlags flags = compiler->flags(); - DCHECK_IMPLIES(IsUnicode(flags), + DCHECK_IMPLIES(IsEitherUnicode(flags), !unibrow::Utf16::IsLeadSurrogate(atom->data().at(0))); bool contains_trail_surrogate = unibrow::Utf16::IsTrailSurrogate(atom->data().at(0)); int first_in_run = i; i++; @@ -777,11 +948,11 @@ while (i < length) { alternative = alternatives->at(i); if (!alternative->IsAtom()) break; RegExpAtom* const alt_atom = alternative->AsAtom(); if (alt_atom->length() != 1) break; - DCHECK_IMPLIES(IsUnicode(flags), + DCHECK_IMPLIES(IsEitherUnicode(flags), !unibrow::Utf16::IsLeadSurrogate(alt_atom->data().at(0))); contains_trail_surrogate |= unibrow::Utf16::IsTrailSurrogate(alt_atom->data().at(0)); i++; } @@ -793,16 +964,16 @@ for (int j = 0; j < run_length; j++) { RegExpAtom* old_atom = alternatives->at(j + first_in_run)->AsAtom(); DCHECK_EQ(old_atom->length(), 1); ranges->Add(CharacterRange::Singleton(old_atom->data().at(0)), zone); } - RegExpCharacterClass::CharacterClassFlags character_class_flags; - if (IsUnicode(flags) && contains_trail_surrogate) { - character_class_flags = RegExpCharacterClass::CONTAINS_SPLIT_SURROGATE; + RegExpClassRanges::ClassRangesFlags class_ranges_flags; + if (IsEitherUnicode(flags) && contains_trail_surrogate) { + class_ranges_flags = RegExpClassRanges::CONTAINS_SPLIT_SURROGATE; } alternatives->at(write_posn++) = - zone->New(zone, ranges, character_class_flags); + zone->New(zone, ranges, class_ranges_flags); } else { // Just copy any trivial alternatives. for (int j = first_in_run; j < i; j++) { alternatives->at(write_posn++) = alternatives->at(j); } @@ -916,12 +1087,12 @@ // Create a newline atom. ZoneList* newline_ranges = zone->New>(3, zone); CharacterRange::AddClassEscape(StandardCharacterSet::kLineTerminator, newline_ranges, false, zone); - RegExpCharacterClass* newline_atom = zone->New( - StandardCharacterSet::kLineTerminator); + RegExpClassRanges* newline_atom = + zone->New(StandardCharacterSet::kLineTerminator); TextNode* newline_matcher = zone->New(newline_atom, false, ActionNode::PositiveSubmatchSuccess( stack_pointer_register, position_register, 0, // No captures inside. @@ -1104,11 +1275,11 @@ // Replace the entire sequence with a single node that always fails. // TODO(jgruber): Consider adding an explicit Fail kind. Until then, the // negated '*' (everything) range serves the purpose. ZoneList* ranges = zone_->New>(0, zone_); - RegExpCharacterClass* cc = zone_->New(zone_, ranges); + RegExpClassRanges* cc = zone_->New(zone_, ranges); terms_->Set(from, cc); // Zero out the rest. RegExpEmpty* empty = zone_->New(); for (int i = from + 1; i < to; i++) terms_->Set(i, empty); @@ -1353,11 +1524,11 @@ } } #endif // V8_INTL_SUPPORT } -bool CharacterRange::IsCanonical(ZoneList* ranges) { +bool CharacterRange::IsCanonical(const ZoneList* ranges) { DCHECK_NOT_NULL(ranges); int n = ranges->length(); if (n <= 1) return true; base::uc32 max = ranges->at(0).to(); for (int i = 1; i < n; i++) { @@ -1494,11 +1665,11 @@ DCHECK(CharacterRange::IsCanonical(character_ranges)); } // static -void CharacterRange::Negate(ZoneList* ranges, +void CharacterRange::Negate(const ZoneList* ranges, ZoneList* negated_ranges, Zone* zone) { DCHECK(CharacterRange::IsCanonical(ranges)); DCHECK_EQ(0, negated_ranges->length()); int range_count = ranges->length(); @@ -1518,10 +1689,135 @@ negated_ranges->Add(CharacterRange::Range(from, kMaxCodePoint), zone); } } // static +void CharacterRange::Intersect(const ZoneList* lhs, + const ZoneList* rhs, + ZoneList* intersection, + Zone* zone) { + DCHECK(CharacterRange::IsCanonical(lhs)); + DCHECK(CharacterRange::IsCanonical(rhs)); + DCHECK_EQ(0, intersection->length()); + int lhs_index = 0; + int rhs_index = 0; + while (lhs_index < lhs->length() && rhs_index < rhs->length()) { + // Skip non-overlapping ranges. + if (lhs->at(lhs_index).to() < rhs->at(rhs_index).from()) { + lhs_index++; + continue; + } + if (rhs->at(rhs_index).to() < lhs->at(lhs_index).from()) { + rhs_index++; + continue; + } + + base::uc32 from = + std::max(lhs->at(lhs_index).from(), rhs->at(rhs_index).from()); + base::uc32 to = std::min(lhs->at(lhs_index).to(), rhs->at(rhs_index).to()); + intersection->Add(CharacterRange::Range(from, to), zone); + if (to == lhs->at(lhs_index).to()) { + lhs_index++; + } else { + rhs_index++; + } + } + + DCHECK(IsCanonical(intersection)); +} + +namespace { + +// Advance |index| and set |from| and |to| to the new range, if not out of +// bounds of |range|, otherwise |from| is set to a code point beyond the legal +// unicode character range. +void SafeAdvanceRange(const ZoneList* range, int* index, + base::uc32* from, base::uc32* to) { + ++(*index); + if (*index < range->length()) { + *from = range->at(*index).from(); + *to = range->at(*index).to(); + } else { + *from = kMaxCodePoint + 1; + } +} + +} // namespace + +// static +void CharacterRange::Subtract(const ZoneList* src, + const ZoneList* to_remove, + ZoneList* result, Zone* zone) { + DCHECK(CharacterRange::IsCanonical(src)); + DCHECK(CharacterRange::IsCanonical(to_remove)); + DCHECK_EQ(0, result->length()); + + if (src->is_empty()) return; + + int src_index = 0; + int to_remove_index = 0; + base::uc32 from = src->at(src_index).from(); + base::uc32 to = src->at(src_index).to(); + while (src_index < src->length() && to_remove_index < to_remove->length()) { + CharacterRange remove_range = to_remove->at(to_remove_index); + if (remove_range.to() < from) { + // (a) Non-overlapping case, ignore current to_remove range. + // |-------| + // |-------| + to_remove_index++; + } else if (to < remove_range.from()) { + // (b) Non-overlapping case, add full current range to result. + // |-------| + // |-------| + result->Add(CharacterRange::Range(from, to), zone); + SafeAdvanceRange(src, &src_index, &from, &to); + } else if (from >= remove_range.from() && to <= remove_range.to()) { + // (c) Current to_remove range fully covers current range. + // |---| + // |-------| + SafeAdvanceRange(src, &src_index, &from, &to); + } else if (from < remove_range.from() && to > remove_range.to()) { + // (d) Split current range. + // |-------| + // |---| + result->Add(CharacterRange::Range(from, remove_range.from() - 1), zone); + from = remove_range.to() + 1; + to_remove_index++; + } else if (from < remove_range.from()) { + // (e) End current range. + // |-------| + // |-------| + to = remove_range.from() - 1; + result->Add(CharacterRange::Range(from, to), zone); + SafeAdvanceRange(src, &src_index, &from, &to); + } else if (to > remove_range.to()) { + // (f) Modify start of current range. + // |-------| + // |-------| + from = remove_range.to() + 1; + to_remove_index++; + } else { + UNREACHABLE(); + } + } + // The last range needs special treatment after |to_remove| is exhausted, as + // |from| might have been modified by the last |to_remove| range and |to| was + // not yet known (i.e. cases d and f). + if (from <= to) { + result->Add(CharacterRange::Range(from, to), zone); + } + src_index++; + + // Add remaining ranges after |to_remove| is exhausted. + for (; src_index < src->length(); src_index++) { + result->Add(src->at(src_index), zone); + } + + DCHECK(IsCanonical(result)); +} + +// static void CharacterRange::ClampToOneByte(ZoneList* ranges) { DCHECK(IsCanonical(ranges)); // Drop all ranges that don't contain one-byte code units, and clamp the last // range s.t. it likewise only contains one-byte code units. Note this relies @@ -1538,10 +1834,24 @@ } ranges->Rewind(n); } +// static +bool CharacterRange::Equals(const ZoneList* lhs, + const ZoneList* rhs) { + DCHECK(IsCanonical(lhs)); + DCHECK(IsCanonical(rhs)); + if (lhs->length() != rhs->length()) return false; + + for (int i = 0; i < lhs->length(); i++) { + if (lhs->at(i) != rhs->at(i)) return false; + } + + return true; +} + namespace { // Scoped object to keep track of how much we unroll quantifier loops in the // regexp graph generator. class RegExpExpansionLimiter { diff --git a/js/src/irregexp/imported/regexp-compiler.h b/js/src/irregexp/imported/regexp-compiler.h --- a/js/src/irregexp/imported/regexp-compiler.h +++ b/js/src/irregexp/imported/regexp-compiler.h @@ -46,13 +46,13 @@ constexpr int kPatternTooShortForBoyerMoore = 2; } // namespace regexp_compiler_constants inline bool NeedsUnicodeCaseEquivalents(RegExpFlags flags) { - // Both unicode and ignore_case flags are set. We need to use ICU to find - // the closure over case equivalents. - return IsUnicode(flags) && IsIgnoreCase(flags); + // Both unicode (or unicode sets) and ignore_case flags are set. We need to + // use ICU to find the closure over case equivalents. + return IsEitherUnicode(flags) && IsIgnoreCase(flags); } // Details of a quick mask-compare check that can look ahead in the // input stream. class QuickCheckDetails { diff --git a/js/src/irregexp/imported/regexp-compiler.cc b/js/src/irregexp/imported/regexp-compiler.cc --- a/js/src/irregexp/imported/regexp-compiler.cc +++ b/js/src/irregexp/imported/regexp-compiler.cc @@ -189,12 +189,12 @@ void RegExpAtom::AppendToText(RegExpText* text, Zone* zone) { text->AddElement(TextElement::Atom(this), zone); } -void RegExpCharacterClass::AppendToText(RegExpText* text, Zone* zone) { - text->AddElement(TextElement::CharClass(this), zone); +void RegExpClassRanges::AppendToText(RegExpText* text, Zone* zone) { + text->AddElement(TextElement::ClassRanges(this), zone); } void RegExpText::AppendToText(RegExpText* text, Zone* zone) { for (int i = 0; i < elements()->length(); i++) text->AddElement(elements()->at(i), zone); @@ -202,20 +202,20 @@ TextElement TextElement::Atom(RegExpAtom* atom) { return TextElement(ATOM, atom); } -TextElement TextElement::CharClass(RegExpCharacterClass* char_class) { - return TextElement(CHAR_CLASS, char_class); +TextElement TextElement::ClassRanges(RegExpClassRanges* class_ranges) { + return TextElement(CLASS_RANGES, class_ranges); } int TextElement::length() const { switch (text_type()) { case ATOM: return atom()->length(); - case CHAR_CLASS: + case CLASS_RANGES: return 1; } UNREACHABLE(); } @@ -241,11 +241,11 @@ recursion_depth_(0), flags_(flags), one_byte_(one_byte), reg_exp_too_big_(false), limiting_recursion_(false), - optimize_(FLAG_regexp_optimization), + optimize_(v8_flags.regexp_optimization), read_backward_(false), current_expansion_factor_(1), frequency_collator_(), isolate_(isolate), zone_(zone) { @@ -271,11 +271,11 @@ work_list.pop_back(); node->set_on_work_list(false); if (!node->label()->is_bound()) node->Emit(this, &new_trace); } if (reg_exp_too_big_) { - if (FLAG_correctness_fuzzer_suppressions) { + if (v8_flags.correctness_fuzzer_suppressions) { FATAL("Aborting on excess zone allocation"); } macro_assembler_->AbortedCodeGeneration(); return CompilationResult::RegExpTooBig(); } @@ -1208,35 +1208,35 @@ &dummy, flip ? odd_label : even_label, flip ? even_label : odd_label); } } -void EmitCharClass(RegExpMacroAssembler* macro_assembler, - RegExpCharacterClass* cc, bool one_byte, Label* on_failure, - int cp_offset, bool check_offset, bool preloaded, - Zone* zone) { - ZoneList* ranges = cc->ranges(zone); +void EmitClassRanges(RegExpMacroAssembler* macro_assembler, + RegExpClassRanges* cr, bool one_byte, Label* on_failure, + int cp_offset, bool check_offset, bool preloaded, + Zone* zone) { + ZoneList* ranges = cr->ranges(zone); CharacterRange::Canonicalize(ranges); // Now that all processing (like case-insensitivity) is done, clamp the // ranges to the set of ranges that may actually occur in the subject string. if (one_byte) CharacterRange::ClampToOneByte(ranges); const int ranges_length = ranges->length(); if (ranges_length == 0) { - if (!cc->is_negated()) { + if (!cr->is_negated()) { macro_assembler->GoTo(on_failure); } if (check_offset) { macro_assembler->CheckPosition(cp_offset, on_failure); } return; } const base::uc32 max_char = MaxCodeUnit(one_byte); if (ranges_length == 1 && ranges->at(0).IsEverything(max_char)) { - if (cc->is_negated()) { + if (cr->is_negated()) { macro_assembler->GoTo(on_failure); } else { // This is a common case hit by non-anchored expressions. if (check_offset) { macro_assembler->CheckPosition(cp_offset, on_failure); @@ -1247,23 +1247,23 @@ if (!preloaded) { macro_assembler->LoadCurrentCharacter(cp_offset, on_failure, check_offset); } - if (cc->is_standard(zone) && macro_assembler->CheckSpecialCharacterClass( - cc->standard_type(), on_failure)) { + if (cr->is_standard(zone) && macro_assembler->CheckSpecialClassRanges( + cr->standard_type(), on_failure)) { return; } static constexpr int kMaxRangesForInlineBranchGeneration = 16; if (ranges_length > kMaxRangesForInlineBranchGeneration) { // For large range sets, emit a more compact instruction sequence to avoid // a potentially problematic increase in code size. // Note the flipped logic below (we check InRange if negated, NotInRange if // not negated); this is necessary since the method falls through on // failure whereas we want to fall through on success. - if (cc->is_negated()) { + if (cr->is_negated()) { if (macro_assembler->CheckCharacterInRangeArray(ranges, on_failure)) { return; } } else { if (macro_assembler->CheckCharacterNotInRangeArray(ranges, on_failure)) { @@ -1276,11 +1276,11 @@ // GenerateBranches. See the comment on that function for how the list should // be structured ZoneList* range_boundaries = zone->New>(ranges_length * 2, zone); - bool zeroth_entry_is_failure = !cc->is_negated(); + bool zeroth_entry_is_failure = !cr->is_negated(); for (int i = 0; i < ranges_length; i++) { CharacterRange& range = ranges->at(i); if (range.from() == 0) { DCHECK_EQ(i, 0); @@ -1652,11 +1652,11 @@ } } } else { QuickCheckDetails::Position* pos = details->positions(characters_filled_in); - RegExpCharacterClass* tree = elm.char_class(); + RegExpClassRanges* tree = elm.class_ranges(); ZoneList* ranges = tree->ranges(zone()); if (tree->is_negated() || ranges->is_empty()) { // A quick check uses multi-character mask and compare. There is no // useful way to incorporate a negative char class into this scheme // so we just conservatively create a mask and value that will always @@ -1880,17 +1880,17 @@ // Replace quark in case we converted to Latin-1. base::uc16* writable_quarks = const_cast(quarks.begin()); writable_quarks[j] = c; } } else { - DCHECK(elm.text_type() == TextElement::CHAR_CLASS); - RegExpCharacterClass* cc = elm.char_class(); - ZoneList* ranges = cc->ranges(zone()); + DCHECK(elm.text_type() == TextElement::CLASS_RANGES); + RegExpClassRanges* cr = elm.class_ranges(); + ZoneList* ranges = cr->ranges(zone()); CharacterRange::Canonicalize(ranges); // Now they are in order so we only need to look at the first. int range_count = ranges->length(); - if (cc->is_negated()) { + if (cr->is_negated()) { if (range_count != 0 && ranges->at(0).from() == 0 && ranges->at(0).to() >= String::kMaxOneByteCharCode) { // This will be handled in a later filter. if (IsIgnoreCase(flags) && RangesContainLatin1Equivalents(ranges)) { continue; @@ -2084,11 +2084,11 @@ namespace { // Check for [0-9A-Z_a-z]. void EmitWordCheck(RegExpMacroAssembler* assembler, Label* word, Label* non_word, bool fall_through_on_word) { - if (assembler->CheckSpecialCharacterClass( + if (assembler->CheckSpecialClassRanges( fall_through_on_word ? StandardCharacterSet::kWord : StandardCharacterSet::kNotWord, fall_through_on_word ? non_word : word)) { // Optimized implementation available. return; @@ -2132,12 +2132,12 @@ // If we've already checked that we are not at the start of input, it's okay // to load the previous character without bounds checks. const bool can_skip_bounds_check = !may_be_at_or_before_subject_string_start; assembler->LoadCurrentCharacter(new_trace.cp_offset() - 1, new_trace.backtrack(), can_skip_bounds_check); - if (!assembler->CheckSpecialCharacterClass( - StandardCharacterSet::kLineTerminator, new_trace.backtrack())) { + if (!assembler->CheckSpecialClassRanges(StandardCharacterSet::kLineTerminator, + new_trace.backtrack())) { // Newline means \n, \r, 0x2028 or 0x2029. if (!compiler->one_byte()) { assembler->CheckCharacterAfterAnd(0x2028, 0xFFFE, &ok); } assembler->CheckCharacter('\n', &ok); @@ -2381,18 +2381,18 @@ break; } if (bounds_checked) UpdateBoundsCheck(cp_offset + j, checked_up_to); } } else { - DCHECK_EQ(TextElement::CHAR_CLASS, elm.text_type()); + DCHECK_EQ(TextElement::CLASS_RANGES, elm.text_type()); if (pass == CHARACTER_CLASS_MATCH) { if (first_element_checked && i == 0) continue; if (DeterminedAlready(quick_check, elm.cp_offset())) continue; - RegExpCharacterClass* cc = elm.char_class(); + RegExpClassRanges* cr = elm.class_ranges(); bool bounds_check = *checked_up_to < cp_offset || read_backward(); - EmitCharClass(assembler, cc, one_byte, backtrack, cp_offset, - bounds_check, preloaded, zone()); + EmitClassRanges(assembler, cr, one_byte, backtrack, cp_offset, + bounds_check, preloaded, zone()); UpdateBoundsCheck(cp_offset, checked_up_to); } } } } @@ -2415,39 +2415,39 @@ ZoneList* ranges, bool read_backward, RegExpNode* on_success) { DCHECK_NOT_NULL(ranges); // TODO(jgruber): There's no fundamental need to create this - // RegExpCharacterClass; we could refactor to avoid the allocation. - return zone->New(zone->New(zone, ranges), + // RegExpClassRanges; we could refactor to avoid the allocation. + return zone->New(zone->New(zone, ranges), read_backward, on_success); } TextNode* TextNode::CreateForSurrogatePair( Zone* zone, CharacterRange lead, ZoneList* trail_ranges, bool read_backward, RegExpNode* on_success) { ZoneList* lead_ranges = CharacterRange::List(zone, lead); ZoneList* elms = zone->New>(2, zone); - elms->Add(TextElement::CharClass( - zone->New(zone, lead_ranges)), - zone); - elms->Add(TextElement::CharClass( - zone->New(zone, trail_ranges)), + elms->Add( + TextElement::ClassRanges(zone->New(zone, lead_ranges)), + zone); + elms->Add(TextElement::ClassRanges( + zone->New(zone, trail_ranges)), zone); return zone->New(elms, read_backward, on_success); } TextNode* TextNode::CreateForSurrogatePair( Zone* zone, ZoneList* lead_ranges, CharacterRange trail, bool read_backward, RegExpNode* on_success) { ZoneList* trail_ranges = CharacterRange::List(zone, trail); ZoneList* elms = zone->New>(2, zone); - elms->Add(TextElement::CharClass( - zone->New(zone, lead_ranges)), - zone); - elms->Add(TextElement::CharClass( - zone->New(zone, trail_ranges)), + elms->Add( + TextElement::ClassRanges(zone->New(zone, lead_ranges)), + zone); + elms->Add(TextElement::ClassRanges( + zone->New(zone, trail_ranges)), zone); return zone->New(elms, read_backward, on_success); } // This generates the code to match a text node. A text node can contain @@ -2527,16 +2527,16 @@ #endif int element_count = elements()->length(); for (int i = 0; i < element_count; i++) { TextElement elm = elements()->at(i); - if (elm.text_type() == TextElement::CHAR_CLASS) { - RegExpCharacterClass* cc = elm.char_class(); + if (elm.text_type() == TextElement::CLASS_RANGES) { + RegExpClassRanges* cr = elm.class_ranges(); // None of the standard character classes is different in the case // independent case and it slows us down if we don't know that. - if (cc->is_standard(zone())) continue; - ZoneList* ranges = cc->ranges(zone()); + if (cr->is_standard(zone())) continue; + ZoneList* ranges = cr->ranges(zone()); CharacterRange::AddCaseEquivalents(isolate, zone(), ranges, is_one_byte); } } } @@ -2545,12 +2545,12 @@ RegExpNode* TextNode::GetSuccessorOfOmnivorousTextNode( RegExpCompiler* compiler) { if (read_backward()) return nullptr; if (elements()->length() != 1) return nullptr; TextElement elm = elements()->at(0); - if (elm.text_type() != TextElement::CHAR_CLASS) return nullptr; - RegExpCharacterClass* node = elm.char_class(); + if (elm.text_type() != TextElement::CLASS_RANGES) return nullptr; + RegExpClassRanges* node = elm.class_ranges(); ZoneList* ranges = node->ranges(zone()); CharacterRange::Canonicalize(ranges); if (node->is_negated()) { return ranges->length() == 0 ? on_success() : nullptr; } @@ -3472,22 +3472,22 @@ RecursionCheck rc(compiler); DCHECK_EQ(start_reg_ + 1, end_reg_); if (IsIgnoreCase(flags_)) { - bool unicode = IsUnicode(flags_); + bool unicode = IsEitherUnicode(flags_); assembler->CheckNotBackReferenceIgnoreCase(start_reg_, read_backward(), unicode, trace->backtrack()); } else { assembler->CheckNotBackReference(start_reg_, read_backward(), trace->backtrack()); } // We are going to advance backward, so we may end up at the start. if (read_backward()) trace->set_at_start(Trace::UNKNOWN); // Check that the back reference does not end inside a surrogate pair. - if (IsUnicode(flags_) && !compiler->one_byte()) { + if (IsEitherUnicode(flags_) && !compiler->one_byte()) { assembler->CheckNotInSurrogatePair(trace->cp_offset(), trace->backtrack()); } on_success()->Emit(compiler, trace); } @@ -3670,11 +3670,11 @@ error_(RegExpError::kNone) {} void EnsureAnalyzed(RegExpNode* that) { StackLimitCheck check(isolate()); if (check.HasOverflowed()) { - if (FLAG_correctness_fuzzer_suppressions) { + if (v8_flags.correctness_fuzzer_suppressions) { FATAL("Analysis: Aborting on stack overflow"); } fail(RegExpError::kAnalysisStackOverflow); return; } @@ -3849,14 +3849,14 @@ } else { if (character <= max_char) bm->Set(offset, character); } } } else { - DCHECK_EQ(TextElement::CHAR_CLASS, text.text_type()); - RegExpCharacterClass* char_class = text.char_class(); - ZoneList* ranges = char_class->ranges(zone()); - if (char_class->is_negated()) { + DCHECK_EQ(TextElement::CLASS_RANGES, text.text_type()); + RegExpClassRanges* class_ranges = text.class_ranges(); + ZoneList* ranges = class_ranges->ranges(zone()); + if (class_ranges->is_negated()) { bm->SetAll(offset); } else { for (int k = 0; k < ranges->length(); k++) { CharacterRange& range = ranges->at(k); if (static_cast(range.from()) > max_char) continue; @@ -3912,20 +3912,20 @@ if (!data->tree->IsAnchoredAtStart() && !IsSticky(flags)) { // Add a .*? at the beginning, outside the body capture, unless // this expression is anchored at the beginning or sticky. RegExpNode* loop_node = RegExpQuantifier::ToNode( 0, RegExpTree::kInfinity, false, - zone()->New(StandardCharacterSet::kEverything), - this, captured_body, data->contains_anchor); + zone()->New(StandardCharacterSet::kEverything), this, + captured_body, data->contains_anchor); if (data->contains_anchor) { // Unroll loop once, to take care of the case that might start // at the start of input. ChoiceNode* first_step_node = zone()->New(2, zone()); first_step_node->AddAlternative(GuardedAlternative(captured_body)); first_step_node->AddAlternative(GuardedAlternative(zone()->New( - zone()->New(StandardCharacterSet::kEverything), + zone()->New(StandardCharacterSet::kEverything), false, loop_node))); node = first_step_node; } else { node = loop_node; } @@ -3935,11 +3935,11 @@ // Do it again to propagate the new nodes to places where they were not // put because they had not been calculated yet. if (node != nullptr) { node = node->FilterOneByte(RegExpCompiler::kMaxRecursion, flags); } - } else if (IsUnicode(flags) && (IsGlobal(flags) || IsSticky(flags))) { + } else if (IsEitherUnicode(flags) && (IsGlobal(flags) || IsSticky(flags))) { node = OptionallyStepBackToLeadSurrogate(node); } if (node == nullptr) node = zone()->New(EndNode::BACKTRACK, zone()); return node; diff --git a/js/src/irregexp/imported/regexp-dotprinter.cc b/js/src/irregexp/imported/regexp-dotprinter.cc --- a/js/src/irregexp/imported/regexp-dotprinter.cc +++ b/js/src/irregexp/imported/regexp-dotprinter.cc @@ -131,12 +131,12 @@ for (int j = 0; j < data.length(); j++) { os_ << static_cast(data[j]); } break; } - case TextElement::CHAR_CLASS: { - RegExpCharacterClass* node = elm.char_class(); + case TextElement::CLASS_RANGES: { + RegExpClassRanges* node = elm.class_ranges(); os_ << "["; if (node->is_negated()) os_ << "^"; for (int j = 0; j < node->ranges(zone)->length(); j++) { CharacterRange range = node->ranges(zone)->at(j); os_ << AsUC32(range.from()) << "-" << AsUC32(range.to()); diff --git a/js/src/irregexp/imported/regexp-error.h b/js/src/irregexp/imported/regexp-error.h --- a/js/src/irregexp/imported/regexp-error.h +++ b/js/src/irregexp/imported/regexp-error.h @@ -39,11 +39,15 @@ T(InvalidNamedCaptureReference, "Invalid named capture referenced") \ T(InvalidClassEscape, "Invalid class escape") \ T(InvalidClassPropertyName, "Invalid property name in character class") \ T(InvalidCharacterClass, "Invalid character class") \ T(UnterminatedCharacterClass, "Unterminated character class") \ - T(OutOfOrderCharacterClass, "Range out of order in character class") + T(OutOfOrderCharacterClass, "Range out of order in character class") \ + T(InvalidClassSetOperation, "Invalid set operation in character class") \ + T(InvalidCharacterInClass, "Invalid character in character class") \ + T(NegatedCharacterClassWithStrings, \ + "Negated character class may contain strings") enum class RegExpError : uint32_t { #define TEMPLATE(NAME, STRING) k##NAME, REGEXP_ERROR_MESSAGES(TEMPLATE) #undef TEMPLATE diff --git a/js/src/irregexp/imported/regexp-interpreter.cc b/js/src/irregexp/imported/regexp-interpreter.cc --- a/js/src/irregexp/imported/regexp-interpreter.cc +++ b/js/src/irregexp/imported/regexp-interpreter.cc @@ -67,11 +67,11 @@ #ifdef DEBUG void MaybeTraceInterpreter(const byte* code_base, const byte* pc, int stack_depth, int current_position, uint32_t current_char, int bytecode_length, const char* bytecode_name) { - if (FLAG_trace_regexp_bytecodes) { + if (v8_flags.trace_regexp_bytecodes) { const bool printable = std::isprint(current_char); const char* format = printable ? "pc = %02x, sp = %d, curpos = %d, curchar = %08x (%c), bc = " : "pc = %02x, sp = %d, curpos = %d, curchar = %08x .%c., bc = "; @@ -436,11 +436,11 @@ BacktrackStack backtrack_stack; uint32_t backtrack_count = 0; #ifdef DEBUG - if (FLAG_trace_regexp_bytecodes) { + if (v8_flags.trace_regexp_bytecodes) { PrintF("\n\nStart bytecode interpreter\n\n"); } #endif while (true) { @@ -1046,11 +1046,11 @@ // static IrregexpInterpreter::Result IrregexpInterpreter::Match( Isolate* isolate, JSRegExp regexp, String subject_string, int* output_registers, int output_register_count, int start_position, RegExp::CallOrigin call_origin) { - if (FLAG_regexp_tier_up) regexp.TierUpTick(); + if (v8_flags.regexp_tier_up) regexp.TierUpTick(); bool is_one_byte = String::IsOneByteRepresentationUnderneath(subject_string); ByteArray code_array = ByteArray::cast(regexp.bytecode(is_one_byte)); int total_register_count = regexp.max_register_count(); diff --git a/js/src/irregexp/imported/regexp-macro-assembler-tracer.h b/js/src/irregexp/imported/regexp-macro-assembler-tracer.h --- a/js/src/irregexp/imported/regexp-macro-assembler-tracer.h +++ b/js/src/irregexp/imported/regexp-macro-assembler-tracer.h @@ -51,12 +51,12 @@ Label* on_in_range) override; bool CheckCharacterNotInRangeArray(const ZoneList* ranges, Label* on_not_in_range) override; void CheckBitInTable(Handle table, Label* on_bit_set) override; void CheckPosition(int cp_offset, Label* on_outside_input) override; - bool CheckSpecialCharacterClass(StandardCharacterSet type, - Label* on_no_match) override; + bool CheckSpecialClassRanges(StandardCharacterSet type, + Label* on_no_match) override; void Fail() override; Handle GetCode(Handle source) override; void GoTo(Label* label) override; void IfRegisterGE(int reg, int comparand, Label* if_ge) override; void IfRegisterLT(int reg, int comparand, Label* if_lt) override; diff --git a/js/src/irregexp/imported/regexp-macro-assembler-tracer.cc b/js/src/irregexp/imported/regexp-macro-assembler-tracer.cc --- a/js/src/irregexp/imported/regexp-macro-assembler-tracer.cc +++ b/js/src/irregexp/imported/regexp-macro-assembler-tracer.cc @@ -388,15 +388,14 @@ PrintF(" CheckPosition(cp_offset=%d, label[%08x]);\n", cp_offset, LabelToInt(on_outside_input)); assembler_->CheckPosition(cp_offset, on_outside_input); } -bool RegExpMacroAssemblerTracer::CheckSpecialCharacterClass( +bool RegExpMacroAssemblerTracer::CheckSpecialClassRanges( StandardCharacterSet type, Label* on_no_match) { - bool supported = assembler_->CheckSpecialCharacterClass(type, - on_no_match); - PrintF(" CheckSpecialCharacterClass(type='%c', label[%08x]): %s;\n", + bool supported = assembler_->CheckSpecialClassRanges(type, on_no_match); + PrintF(" CheckSpecialClassRanges(type='%c', label[%08x]): %s;\n", static_cast(type), LabelToInt(on_no_match), supported ? "true" : "false"); return supported; } diff --git a/js/src/irregexp/imported/regexp-macro-assembler.h b/js/src/irregexp/imported/regexp-macro-assembler.h --- a/js/src/irregexp/imported/regexp-macro-assembler.h +++ b/js/src/irregexp/imported/regexp-macro-assembler.h @@ -111,12 +111,12 @@ virtual void CheckPosition(int cp_offset, Label* on_outside_input); // Check whether a standard/default character class matches the current // character. Returns false if the type of special character class does // not have custom support. // May clobber the current loaded character. - virtual bool CheckSpecialCharacterClass(StandardCharacterSet type, - Label* on_no_match) { + virtual bool CheckSpecialClassRanges(StandardCharacterSet type, + Label* on_no_match) { return false; } // Control-flow integrity: // Define a jump target and bind a label. diff --git a/js/src/irregexp/imported/regexp-macro-assembler.cc b/js/src/irregexp/imported/regexp-macro-assembler.cc --- a/js/src/irregexp/imported/regexp-macro-assembler.cc +++ b/js/src/irregexp/imported/regexp-macro-assembler.cc @@ -266,11 +266,11 @@ } LoadCurrentCharacterUnchecked(cp_offset, characters); } bool NativeRegExpMacroAssembler::CanReadUnaligned() const { - return FLAG_enable_regexp_unaligned_accesses && !slow_safe(); + return v8_flags.enable_regexp_unaligned_accesses && !slow_safe(); } #ifndef COMPILING_IRREGEXP_FOR_EXTERNAL_EMBEDDER // This method may only be called after an interrupt. diff --git a/js/src/irregexp/imported/regexp-nodes.h b/js/src/irregexp/imported/regexp-nodes.h --- a/js/src/irregexp/imported/regexp-nodes.h +++ b/js/src/irregexp/imported/regexp-nodes.h @@ -393,16 +393,15 @@ class TextNode : public SeqRegExpNode { public: TextNode(ZoneList* elms, bool read_backward, RegExpNode* on_success) : SeqRegExpNode(on_success), elms_(elms), read_backward_(read_backward) {} - TextNode(RegExpCharacterClass* that, bool read_backward, - RegExpNode* on_success) + TextNode(RegExpClassRanges* that, bool read_backward, RegExpNode* on_success) : SeqRegExpNode(on_success), elms_(zone()->New>(1, zone())), read_backward_(read_backward) { - elms_->Add(TextElement::CharClass(that), zone()); + elms_->Add(TextElement::ClassRanges(that), zone()); } // Create TextNode for a single character class for the given ranges. static TextNode* CreateForCharacterRanges(Zone* zone, ZoneList* ranges, bool read_backward, diff --git a/js/src/irregexp/imported/regexp-parser.cc b/js/src/irregexp/imported/regexp-parser.cc --- a/js/src/irregexp/imported/regexp-parser.cc +++ b/js/src/irregexp/imported/regexp-parser.cc @@ -2,17 +2,18 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "irregexp/imported/regexp-parser.h" -#include "irregexp/imported/property-sequences.h" #include "irregexp/imported/regexp-ast.h" #include "irregexp/imported/regexp-macro-assembler.h" #include "irregexp/imported/regexp.h" #ifdef V8_INTL_SUPPORT #include "unicode/uniset.h" +#include "unicode/unistr.h" +#include "unicode/usetiter.h" #endif // V8_INTL_SUPPORT namespace v8 { namespace internal { @@ -23,27 +24,284 @@ enum class InClassEscapeState { kInClass, kNotInClass, }; +// The production used to derive ClassSetOperand. +enum class ClassSetOperandType { + kClassSetCharacter, + kClassStringDisjunction, + kNestedClass, + kCharacterClassEscape, // \ CharacterClassEscape is a special nested class, + // as we can fold it directly into another range. + kClassSetRange +}; + +class RegExpTextBuilder { + public: + using SmallRegExpTreeVector = + base::SmallVector>; + + RegExpTextBuilder(Zone* zone, SmallRegExpTreeVector* terms_storage, + RegExpFlags flags) + : zone_(zone), + flags_(flags), + terms_(terms_storage), + text_(ZoneAllocator{zone}) {} + void AddCharacter(base::uc16 character); + void AddUnicodeCharacter(base::uc32 character); + void AddEscapedUnicodeCharacter(base::uc32 character); + void AddAtom(RegExpTree* atom); + void AddTerm(RegExpTree* term); + void AddClassRanges(RegExpClassRanges* cc); + void FlushPendingSurrogate(); + void FlushText(); + RegExpTree* PopLastAtom(); + RegExpTree* ToRegExp(); + + private: + static const base::uc16 kNoPendingSurrogate = 0; + + void AddLeadSurrogate(base::uc16 lead_surrogate); + void AddTrailSurrogate(base::uc16 trail_surrogate); + void FlushCharacters(); + bool NeedsDesugaringForUnicode(RegExpClassRanges* cc); + bool NeedsDesugaringForIgnoreCase(base::uc32 c); + void AddClassRangesForDesugaring(base::uc32 c); + bool ignore_case() const { return IsIgnoreCase(flags_); } + bool IsUnicodeMode() const { + // Either /v or /u enable UnicodeMode + // TODO(v8:11935): Change permalink once proposal is in stage 4. + // https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#sec-parsepattern + return IsUnicode(flags_) || IsUnicodeSets(flags_); + } + Zone* zone() const { return zone_; } + + Zone* const zone_; + const RegExpFlags flags_; + ZoneList* characters_ = nullptr; + base::uc16 pending_surrogate_ = kNoPendingSurrogate; + SmallRegExpTreeVector* terms_; + SmallRegExpTreeVector text_; +}; + +void RegExpTextBuilder::AddLeadSurrogate(base::uc16 lead_surrogate) { + DCHECK(unibrow::Utf16::IsLeadSurrogate(lead_surrogate)); + FlushPendingSurrogate(); + // Hold onto the lead surrogate, waiting for a trail surrogate to follow. + pending_surrogate_ = lead_surrogate; +} + +void RegExpTextBuilder::AddTrailSurrogate(base::uc16 trail_surrogate) { + DCHECK(unibrow::Utf16::IsTrailSurrogate(trail_surrogate)); + if (pending_surrogate_ != kNoPendingSurrogate) { + base::uc16 lead_surrogate = pending_surrogate_; + pending_surrogate_ = kNoPendingSurrogate; + DCHECK(unibrow::Utf16::IsLeadSurrogate(lead_surrogate)); + base::uc32 combined = + unibrow::Utf16::CombineSurrogatePair(lead_surrogate, trail_surrogate); + if (NeedsDesugaringForIgnoreCase(combined)) { + AddClassRangesForDesugaring(combined); + } else { + ZoneList surrogate_pair(2, zone()); + surrogate_pair.Add(lead_surrogate, zone()); + surrogate_pair.Add(trail_surrogate, zone()); + RegExpAtom* atom = + zone()->New(surrogate_pair.ToConstVector()); + AddAtom(atom); + } + } else { + pending_surrogate_ = trail_surrogate; + FlushPendingSurrogate(); + } +} + +void RegExpTextBuilder::FlushPendingSurrogate() { + if (pending_surrogate_ != kNoPendingSurrogate) { + DCHECK(IsUnicodeMode()); + base::uc32 c = pending_surrogate_; + pending_surrogate_ = kNoPendingSurrogate; + AddClassRangesForDesugaring(c); + } +} + +void RegExpTextBuilder::FlushCharacters() { + FlushPendingSurrogate(); + if (characters_ != nullptr) { + RegExpTree* atom = zone()->New(characters_->ToConstVector()); + characters_ = nullptr; + text_.emplace_back(atom); + } +} + +void RegExpTextBuilder::FlushText() { + FlushCharacters(); + size_t num_text = text_.size(); + if (num_text == 0) { + return; + } else if (num_text == 1) { + terms_->emplace_back(text_.back()); + } else { + RegExpText* text = zone()->New(zone()); + for (size_t i = 0; i < num_text; i++) { + text_[i]->AppendToText(text, zone()); + } + terms_->emplace_back(text); + } + text_.clear(); +} + +void RegExpTextBuilder::AddCharacter(base::uc16 c) { + FlushPendingSurrogate(); + if (NeedsDesugaringForIgnoreCase(c)) { + AddClassRangesForDesugaring(c); + } else { + if (characters_ == nullptr) { + characters_ = zone()->New>(4, zone()); + } + characters_->Add(c, zone()); + } +} + +void RegExpTextBuilder::AddUnicodeCharacter(base::uc32 c) { + if (c > static_cast(unibrow::Utf16::kMaxNonSurrogateCharCode)) { + DCHECK(IsUnicodeMode()); + AddLeadSurrogate(unibrow::Utf16::LeadSurrogate(c)); + AddTrailSurrogate(unibrow::Utf16::TrailSurrogate(c)); + } else if (IsUnicodeMode() && unibrow::Utf16::IsLeadSurrogate(c)) { + AddLeadSurrogate(c); + } else if (IsUnicodeMode() && unibrow::Utf16::IsTrailSurrogate(c)) { + AddTrailSurrogate(c); + } else { + AddCharacter(static_cast(c)); + } +} + +void RegExpTextBuilder::AddEscapedUnicodeCharacter(base::uc32 character) { + // A lead or trail surrogate parsed via escape sequence will not + // pair up with any preceding lead or following trail surrogate. + FlushPendingSurrogate(); + AddUnicodeCharacter(character); + FlushPendingSurrogate(); +} + +void RegExpTextBuilder::AddClassRanges(RegExpClassRanges* cr) { + if (NeedsDesugaringForUnicode(cr)) { + // With /u or /v, character class needs to be desugared, so it + // must be a standalone term instead of being part of a RegExpText. + AddTerm(cr); + } else { + AddAtom(cr); + } +} + +void RegExpTextBuilder::AddClassRangesForDesugaring(base::uc32 c) { + AddTerm(zone()->New( + zone(), CharacterRange::List(zone(), CharacterRange::Singleton(c)))); +} + +void RegExpTextBuilder::AddAtom(RegExpTree* atom) { + DCHECK(atom->IsTextElement()); + FlushCharacters(); + text_.emplace_back(atom); +} + +void RegExpTextBuilder::AddTerm(RegExpTree* term) { + DCHECK(term->IsTextElement()); + FlushText(); + terms_->emplace_back(term); +} + +bool RegExpTextBuilder::NeedsDesugaringForUnicode(RegExpClassRanges* cc) { + if (!IsUnicodeMode()) return false; + // TODO(yangguo): we could be smarter than this. Case-insensitivity does not + // necessarily mean that we need to desugar. It's probably nicer to have a + // separate pass to figure out unicode desugarings. + if (ignore_case()) return true; + ZoneList* ranges = cc->ranges(zone()); + CharacterRange::Canonicalize(ranges); + + if (cc->is_negated()) { + ZoneList* negated_ranges = + zone()->New>(ranges->length(), zone()); + CharacterRange::Negate(ranges, negated_ranges, zone()); + ranges = negated_ranges; + } + + for (int i = ranges->length() - 1; i >= 0; i--) { + base::uc32 from = ranges->at(i).from(); + base::uc32 to = ranges->at(i).to(); + // Check for non-BMP characters. + if (to >= kNonBmpStart) return true; + // Check for lone surrogates. + if (from <= kTrailSurrogateEnd && to >= kLeadSurrogateStart) return true; + } + return false; +} + +bool RegExpTextBuilder::NeedsDesugaringForIgnoreCase(base::uc32 c) { +#ifdef V8_INTL_SUPPORT + if (IsUnicodeMode() && ignore_case()) { + icu::UnicodeSet set(c, c); + set.closeOver(USET_CASE_INSENSITIVE); + set.removeAllStrings(); + return set.size() > 1; + } + // In the case where ICU is not included, we act as if the unicode flag is + // not set, and do not desugar. +#endif // V8_INTL_SUPPORT + return false; +} + +RegExpTree* RegExpTextBuilder::PopLastAtom() { + FlushPendingSurrogate(); + RegExpTree* atom; + if (characters_ != nullptr) { + base::Vector char_vector = characters_->ToConstVector(); + int num_chars = char_vector.length(); + if (num_chars > 1) { + base::Vector prefix = + char_vector.SubVector(0, num_chars - 1); + text_.emplace_back(zone()->New(prefix)); + char_vector = char_vector.SubVector(num_chars - 1, num_chars); + } + characters_ = nullptr; + atom = zone()->New(char_vector); + return atom; + } else if (text_.size() > 0) { + atom = text_.back(); + text_.pop_back(); + return atom; + } + return nullptr; +} + +RegExpTree* RegExpTextBuilder::ToRegExp() { + FlushText(); + size_t num_alternatives = terms_->size(); + if (num_alternatives == 0) return zone()->New(); + if (num_alternatives == 1) return terms_->back(); + return zone()->New(zone()->New>( + base::VectorOf(terms_->begin(), terms_->size()), zone())); +} + // Accumulates RegExp atoms and assertions into lists of terms and alternatives. class RegExpBuilder { public: RegExpBuilder(Zone* zone, RegExpFlags flags) : zone_(zone), flags_(flags), terms_(ZoneAllocator{zone}), - text_(ZoneAllocator{zone}), - alternatives_(ZoneAllocator{zone}) {} + alternatives_(ZoneAllocator{zone}), + text_builder_(RegExpTextBuilder{zone, &terms_, flags}) {} void AddCharacter(base::uc16 character); void AddUnicodeCharacter(base::uc32 character); void AddEscapedUnicodeCharacter(base::uc32 character); // "Adds" an empty expression. Does nothing except consume a // following quantifier void AddEmpty(); - void AddCharacterClass(RegExpCharacterClass* cc); - void AddCharacterClassForDesugaring(base::uc32 c); + void AddClassRanges(RegExpClassRanges* cc); void AddAtom(RegExpTree* tree); void AddTerm(RegExpTree* tree); void AddAssertion(RegExpTree* tree); void NewAlternative(); // '|' bool AddQuantifierToAtom(int min, int max, @@ -55,44 +313,29 @@ bool ignore_case() const { return IsIgnoreCase(flags_); } bool multiline() const { return IsMultiline(flags_); } bool dotall() const { return IsDotAll(flags_); } private: - static const base::uc16 kNoPendingSurrogate = 0; - void AddLeadSurrogate(base::uc16 lead_surrogate); - void AddTrailSurrogate(base::uc16 trail_surrogate); - void FlushPendingSurrogate(); - void FlushCharacters(); void FlushTerms(); - bool NeedsDesugaringForUnicode(RegExpCharacterClass* cc); - bool NeedsDesugaringForIgnoreCase(base::uc32 c); + bool IsUnicodeMode() const { + // Either /v or /u enable UnicodeMode + // TODO(v8:11935): Change permalink once proposal is in stage 4. + // https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#sec-parsepattern + return IsUnicode(flags_) || IsUnicodeSets(flags_); + } Zone* zone() const { return zone_; } - bool unicode() const { return IsUnicode(flags_); } + RegExpTextBuilder& text_builder() { return text_builder_; } Zone* const zone_; bool pending_empty_ = false; const RegExpFlags flags_; - ZoneList* characters_ = nullptr; - base::uc16 pending_surrogate_ = kNoPendingSurrogate; using SmallRegExpTreeVector = base::SmallVector>; SmallRegExpTreeVector terms_; - SmallRegExpTreeVector text_; SmallRegExpTreeVector alternatives_; -#ifdef DEBUG - enum { - ADD_NONE, - ADD_CHAR, - ADD_TERM, - ADD_ASSERT, - ADD_ATOM - } last_added_ = ADD_NONE; -#define LAST(x) last_added_ = x; -#else -#define LAST(x) -#endif + RegExpTextBuilder text_builder_; }; enum SubexpressionType { INITIAL, CAPTURE, // All positive values represent captures. @@ -197,39 +440,62 @@ bool ParseUnicodeEscape(base::uc32* value); bool ParseUnlimitedLengthHexNumber(int max_value, base::uc32* value); bool ParsePropertyClassName(ZoneVector* name_1, ZoneVector* name_2); - bool AddPropertyClassRange(ZoneList* add_to, bool negate, + bool AddPropertyClassRange(ZoneList* add_to_range, + CharacterClassStrings* add_to_strings, bool negate, const ZoneVector& name_1, const ZoneVector& name_2); + RegExpTree* ParseClassRanges(ZoneList* ranges, + bool add_unicode_case_equivalents); + // Parse inside a class. Either add escaped class to the range, or return + // false and pass parsed single character through |char_out|. + void ParseClassEscape(ZoneList* ranges, Zone* zone, + bool add_unicode_case_equivalents, base::uc32* char_out, + bool* is_class_escape); + // Returns true iff parsing was successful. + bool TryParseCharacterClassEscape(base::uc32 next, + InClassEscapeState in_class_escape_state, + ZoneList* ranges, + CharacterClassStrings* strings, Zone* zone, + bool add_unicode_case_equivalents); + RegExpTree* ParseClassStringDisjunction(ZoneList* ranges, + CharacterClassStrings* strings); + RegExpTree* ParseClassSetOperand(const RegExpBuilder* builder, + ClassSetOperandType* type_out); + RegExpTree* ParseClassSetOperand(const RegExpBuilder* builder, + ClassSetOperandType* type_out, + ZoneList* ranges, + CharacterClassStrings* strings); + base::uc32 ParseClassSetCharacter(); + // Parses and returns a single escaped character. + base::uc32 ParseCharacterEscape(InClassEscapeState in_class_escape_state, + bool* is_escaped_unicode_character); + + RegExpTree* ParseClassUnion(const RegExpBuilder* builder, bool is_negated, + RegExpTree* first_operand, + ClassSetOperandType first_operand_type, + ZoneList* ranges, + CharacterClassStrings* strings); + RegExpTree* ParseClassIntersection(const RegExpBuilder* builder, + bool is_negated, RegExpTree* first_operand, + ClassSetOperandType first_operand_type); + RegExpTree* ParseClassSubtraction(const RegExpBuilder* builder, + bool is_negated, RegExpTree* first_operand, + ClassSetOperandType first_operand_type); RegExpTree* ParseCharacterClass(const RegExpBuilder* state); base::uc32 ParseOctalLiteral(); // Tries to parse the input as a back reference. If successful it // stores the result in the output parameter and returns true. If // it fails it will push back the characters read so the same characters // can be reparsed. bool ParseBackReferenceIndex(int* index_out); - // Parse inside a class. Either add escaped class to the range, or return - // false and pass parsed single character through |char_out|. - void ParseClassEscape(ZoneList* ranges, Zone* zone, - bool add_unicode_case_equivalents, base::uc32* char_out, - bool* is_class_escape); - // Returns true iff parsing was successful. - bool TryParseCharacterClassEscape(base::uc32 next, - InClassEscapeState in_class_escape_state, - ZoneList* ranges, - Zone* zone, - bool add_unicode_case_equivalents); - // Parses and returns a single escaped character. - base::uc32 ParseCharacterEscape(InClassEscapeState in_class_escape_state, - bool* is_escaped_unicode_character); - RegExpTree* ReportError(RegExpError error); void Advance(); void Advance(int dist); void RewindByOneCodepoint(); // Rewinds to before the previous Advance(). void Reset(int pos); @@ -240,13 +506,24 @@ bool contains_anchor() const { return contains_anchor_; } void set_contains_anchor() { contains_anchor_ = true; } int captures_started() const { return captures_started_; } int position() const { return next_pos_ - 1; } bool failed() const { return failed_; } - bool unicode() const { return IsUnicode(top_level_flags_) || force_unicode_; } + RegExpFlags flags() const { return top_level_flags_; } + bool IsUnicodeMode() const { + // Either /v or /u enable UnicodeMode + // TODO(v8:11935): Change permalink once proposal is in stage 4. + // https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#sec-parsepattern + return IsUnicode(flags()) || IsUnicodeSets(flags()) || force_unicode_; + } + bool unicode_sets() const { return IsUnicodeSets(flags()); } + bool ignore_case() const { return IsIgnoreCase(flags()); } static bool IsSyntaxCharacterOrSlash(base::uc32 c); + static bool IsClassSetSyntaxCharacter(base::uc32 c); + static bool IsClassSetReservedPunctuator(base::uc32 c); + bool IsClassSetReservedDoublePunctuator(base::uc32 c); static const base::uc32 kEndMarker = (1 << 21); private: // Return the 1-indexed RegExpCapture object, allocate if necessary. @@ -382,12 +659,12 @@ inline base::uc32 RegExpParserImpl::ReadNext() { int position = next_pos_; base::uc16 c0 = InputAt(position); base::uc32 result = c0; position++; - // Read the whole surrogate pair in case of unicode flag, if possible. - if (unicode() && position < input_length() && + // Read the whole surrogate pair in case of unicode mode, if possible. + if (IsUnicodeMode() && position < input_length() && unibrow::Utf16::IsLeadSurrogate(c0)) { base::uc16 c1 = InputAt(position); if (unibrow::Utf16::IsTrailSurrogate(c1)) { result = unibrow::Utf16::CombineSurrogatePair(c0, c1); position++; @@ -408,11 +685,11 @@ template void RegExpParserImpl::Advance() { if (has_next()) { if (GetCurrentStackPosition() < stack_limit_) { - if (FLAG_correctness_fuzzer_suppressions) { + if (v8_flags.correctness_fuzzer_suppressions) { FATAL("Aborting on stack overflow"); } ReportError(RegExpError::kStackOverflow); } else { current_ = ReadNext(); @@ -426,11 +703,11 @@ } } template void RegExpParserImpl::RewindByOneCodepoint() { - if (current() == kEndMarker) return; + if (!has_more()) return; // Rewinds by one code point, i.e.: two code units if `current` is outside // the basic multilingual plane (= composed of a lead and trail surrogate), // or one code unit otherwise. const int rewind_by = current() > unibrow::Utf16::kMaxNonSurrogateCharCode ? -2 : -1; @@ -448,10 +725,11 @@ void RegExpParserImpl::Advance(int dist) { next_pos_ += dist - 1; Advance(); } +// static template bool RegExpParserImpl::IsSyntaxCharacterOrSlash(base::uc32 c) { switch (c) { case '^': case '$': @@ -473,19 +751,100 @@ break; } return false; } +// static +template +bool RegExpParserImpl::IsClassSetSyntaxCharacter(base::uc32 c) { + switch (c) { + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '/': + case '-': + case '\\': + case '|': + return true; + default: + break; + } + return false; +} + +// static +template +bool RegExpParserImpl::IsClassSetReservedPunctuator(base::uc32 c) { + switch (c) { + case '&': + case '-': + case '!': + case '#': + case '%': + case ',': + case ':': + case ';': + case '<': + case '=': + case '>': + case '@': + case '`': + case '~': + return true; + default: + break; + } + return false; +} + +template +bool RegExpParserImpl::IsClassSetReservedDoublePunctuator(base::uc32 c) { +#define DOUBLE_PUNCTUATOR_CASE(Char) \ + case Char: \ + return Next() == Char + + switch (c) { + DOUBLE_PUNCTUATOR_CASE('&'); + DOUBLE_PUNCTUATOR_CASE('!'); + DOUBLE_PUNCTUATOR_CASE('#'); + DOUBLE_PUNCTUATOR_CASE('$'); + DOUBLE_PUNCTUATOR_CASE('%'); + DOUBLE_PUNCTUATOR_CASE('*'); + DOUBLE_PUNCTUATOR_CASE('+'); + DOUBLE_PUNCTUATOR_CASE(','); + DOUBLE_PUNCTUATOR_CASE('.'); + DOUBLE_PUNCTUATOR_CASE(':'); + DOUBLE_PUNCTUATOR_CASE(';'); + DOUBLE_PUNCTUATOR_CASE('<'); + DOUBLE_PUNCTUATOR_CASE('='); + DOUBLE_PUNCTUATOR_CASE('>'); + DOUBLE_PUNCTUATOR_CASE('?'); + DOUBLE_PUNCTUATOR_CASE('@'); + DOUBLE_PUNCTUATOR_CASE('^'); + DOUBLE_PUNCTUATOR_CASE('`'); + DOUBLE_PUNCTUATOR_CASE('~'); + default: + break; + } +#undef DOUBLE_PUNCTUATOR_CASE + + return false; +} + template RegExpTree* RegExpParserImpl::ReportError(RegExpError error) { if (failed_) return nullptr; // Do not overwrite any existing error. failed_ = true; error_ = error; error_pos_ = position(); // Zip to the end to make sure no more input is read. current_ = kEndMarker; next_pos_ = input_length(); + has_more_ = false; return nullptr; } #define CHECK_FAILED /**/); \ if (failed_) return nullptr; \ @@ -518,11 +877,11 @@ // Atom Quantifier template RegExpTree* RegExpParserImpl::ParseDisjunction() { // Used to store current state while parsing subexpressions. RegExpParserState initial_state(nullptr, INITIAL, RegExpLookaround::LOOKAHEAD, - 0, nullptr, top_level_flags_, zone()); + 0, nullptr, flags(), zone()); RegExpParserState* state = &initial_state; // Cache the builder in a local variable for quick access. RegExpBuilder* builder = initial_state.builder(); while (true) { switch (current()) { @@ -619,23 +978,28 @@ // Everything except \x0A, \x0D, \u2028 and \u2029. CharacterRange::AddClassEscape( StandardCharacterSet::kNotLineTerminator, ranges, false, zone()); } - RegExpCharacterClass* cc = - zone()->template New(zone(), ranges); - builder->AddCharacterClass(cc); + RegExpClassRanges* cc = + zone()->template New(zone(), ranges); + builder->AddClassRanges(cc); break; } case '(': { state = ParseOpenParenthesis(state CHECK_FAILED); builder = state->builder(); continue; } case '[': { RegExpTree* cc = ParseCharacterClass(builder CHECK_FAILED); - builder->AddCharacterClass(cc->AsCharacterClass()); + if (cc->IsClassRanges()) { + builder->AddClassRanges(cc->AsClassRanges()); + } else { + DCHECK(cc->IsClassSetExpression()); + builder->AddTerm(cc); + } break; } // Atom :: // \ AtomEscape case '\\': @@ -681,13 +1045,13 @@ capture, builder->flags()); builder->AddAtom(atom); } break; } - // With /u, no identity escapes except for syntax characters + // With /u and /v, no identity escapes except for syntax characters // are allowed. Otherwise, all identity escapes are allowed. - if (unicode()) { + if (IsUnicodeMode()) { return ReportError(RegExpError::kInvalidEscape); } base::uc32 first_digit = Next(); if (first_digit == '8' || first_digit == '9') { builder->AddCharacter(first_digit); @@ -696,12 +1060,12 @@ } V8_FALLTHROUGH; } case '0': { Advance(); - if (unicode() && Next() >= '0' && Next() <= '9') { - // With /u, decimal escape with leading 0 are not parsed as octal. + if (IsUnicodeMode() && Next() >= '0' && Next() <= '9') { + // Decimal escape with leading 0 are not parsed as octal. return ReportError(RegExpError::kInvalidDecimalEscape); } base::uc32 octal = ParseOctalLiteral(); builder->AddCharacter(octal); break; @@ -721,28 +1085,58 @@ case 'd': case 'D': case 's': case 'S': case 'w': - case 'W': + case 'W': { + base::uc32 next = Next(); + ZoneList* ranges = + zone()->template New>(2, zone()); + bool add_unicode_case_equivalents = + IsUnicodeMode() && ignore_case(); + bool parsed_character_class_escape = TryParseCharacterClassEscape( + next, InClassEscapeState::kNotInClass, ranges, nullptr, zone(), + add_unicode_case_equivalents CHECK_FAILED); + + if (parsed_character_class_escape) { + RegExpClassRanges* cc = + zone()->template New(zone(), ranges); + builder->AddClassRanges(cc); + } else { + CHECK(!IsUnicodeMode()); + Advance(2); + builder->AddCharacter(next); // IdentityEscape. + } + break; + } case 'p': case 'P': { base::uc32 next = Next(); ZoneList* ranges = zone()->template New>(2, zone()); - bool add_unicode_case_equivalents = - unicode() && builder->ignore_case(); + CharacterClassStrings* strings = nullptr; + if (unicode_sets()) { + strings = zone()->template New(zone()); + } + bool add_unicode_case_equivalents = ignore_case(); bool parsed_character_class_escape = TryParseCharacterClassEscape( - next, InClassEscapeState::kNotInClass, ranges, zone(), + next, InClassEscapeState::kNotInClass, ranges, strings, zone(), add_unicode_case_equivalents CHECK_FAILED); if (parsed_character_class_escape) { - RegExpCharacterClass* cc = - zone()->template New(zone(), ranges); - builder->AddCharacterClass(cc); + if (unicode_sets()) { + RegExpClassSetOperand* op = + zone()->template New(ranges, + strings); + builder->AddTerm(op); + } else { + RegExpClassRanges* cc = + zone()->template New(zone(), ranges); + builder->AddClassRanges(cc); + } } else { - CHECK(!unicode()); + CHECK(!IsUnicodeMode()); Advance(2); builder->AddCharacter(next); // IdentityEscape. } break; } @@ -754,11 +1148,11 @@ // an identity escape for non-Unicode patterns without named // capture groups, and as the beginning of a named back-reference // in all other cases. const bool has_named_captures = HasNamedCaptures(InClassEscapeState::kNotInClass CHECK_FAILED); - if (unicode() || has_named_captures) { + if (IsUnicodeMode() || has_named_captures) { Advance(2); ParseNamedBackReference(builder, state CHECK_FAILED); break; } } @@ -785,11 +1179,11 @@ if (parsed) return ReportError(RegExpError::kNothingToRepeat); V8_FALLTHROUGH; } case '}': case ']': - if (unicode()) { + if (IsUnicodeMode()) { return ReportError(RegExpError::kLoneQuantifierBrackets); } V8_FALLTHROUGH; default: builder->AddUnicodeCharacter(current()); @@ -824,24 +1218,24 @@ if (ParseIntervalQuantifier(&min, &max)) { if (max < min) { return ReportError(RegExpError::kRangeOutOfOrder); } break; - } else if (unicode()) { - // With /u, incomplete quantifiers are not allowed. + } else if (IsUnicodeMode()) { + // Incomplete quantifiers are not allowed. return ReportError(RegExpError::kIncompleteQuantifier); } continue; default: continue; } RegExpQuantifier::QuantifierType quantifier_type = RegExpQuantifier::GREEDY; if (current() == '?') { quantifier_type = RegExpQuantifier::NON_GREEDY; Advance(); - } else if (FLAG_regexp_possessive_quantifier && current() == '+') { - // FLAG_regexp_possessive_quantifier is a debug-only flag. + } else if (v8_flags.regexp_possessive_quantifier && current() == '+') { + // v8_flags.regexp_possessive_quantifier is a debug-only flag. quantifier_type = RegExpQuantifier::POSSESSIVE; Advance(); } if (!builder->AddQuantifierToAtom(min, max, quantifier_type)) { return ReportError(RegExpError::kInvalidQuantifier); @@ -938,20 +1332,24 @@ // is called when needed. It can see the difference between capturing and // noncapturing parentheses and can skip character classes and backslash-escaped // characters. // // Important: The scanner has to be in a consistent state when calling -// ScanForCaptures, e.g. not in the middle of an escape sequence '\['. +// ScanForCaptures, e.g. not in the middle of an escape sequence '\[' or while +// parsing a nested class. template void RegExpParserImpl::ScanForCaptures( InClassEscapeState in_class_escape_state) { DCHECK(!is_scanned_for_captures_); const int saved_position = position(); // Start with captures started previous to current position int capture_count = captures_started(); // When we start inside a character class, skip everything inside the class. if (in_class_escape_state == InClassEscapeState::kInClass) { + // \k is always invalid within a class in unicode mode, thus we should never + // call ScanForCaptures within a class. + DCHECK(!IsUnicodeMode()); int c; while ((c = current()) != kEndMarker) { Advance(); if (c == '\\') { Advance(); @@ -967,17 +1365,23 @@ switch (n) { case '\\': Advance(); break; case '[': { + int class_nest_level = 0; int c; while ((c = current()) != kEndMarker) { Advance(); if (c == '\\') { Advance(); - } else { - if (c == ']') break; + } else if (c == '[') { + // With /v, '[' inside a class is treated as a nested class. + // Without /v, '[' is a normal character. + if (unicode_sets()) class_nest_level++; + } else if (c == ']') { + if (class_nest_level == 0) break; + class_nest_level--; } } break; } case '(': @@ -1030,12 +1434,13 @@ } else { break; } } if (value > captures_started()) { - if (!is_scanned_for_captures_) + if (!is_scanned_for_captures_) { ScanForCaptures(InClassEscapeState::kNotInClass); + } if (value > capture_count_) { Reset(start); return false; } } @@ -1335,11 +1740,11 @@ return true; } template base::uc32 RegExpParserImpl::ParseOctalLiteral() { - DCHECK(('0' <= current() && current() <= '7') || current() == kEndMarker); + DCHECK(('0' <= current() && current() <= '7') || !has_more()); // For compatibility with some other browsers (not all), we parse // up to three octal digits with a value below 256. // ES#prod-annexB-LegacyOctalEscapeSequence base::uc32 value = current() - '0'; Advance(); @@ -1376,11 +1781,11 @@ template bool RegExpParserImpl::ParseUnicodeEscape(base::uc32* value) { // Accept both \uxxxx and \u{xxxxxx} (if harmony unicode escapes are // allowed). In the latter case, the number of hex digits between { } is // arbitrary. \ and u have already been read. - if (current() == '{' && unicode()) { + if (current() == '{' && IsUnicodeMode()) { int start = position(); Advance(); if (ParseUnlimitedLengthHexNumber(0x10FFFF, value)) { if (current() == '}') { Advance(); @@ -1390,11 +1795,11 @@ Reset(start); return false; } // \u but no {, or \u{...} escapes not allowed. bool result = ParseHexEscape(4, value); - if (result && unicode() && unibrow::Utf16::IsLeadSurrogate(*value) && + if (result && IsUnicodeMode() && unibrow::Utf16::IsLeadSurrogate(*value) && current() == '\\') { // Attempt to read trail surrogate. int start = position(); if (Next() == 'u') { Advance(2); @@ -1443,13 +1848,48 @@ if (strcmp(property_value_name, long_name) == 0) return true; } return false; } +void ExtractStringsFromUnicodeSet(const icu::UnicodeSet& set, + CharacterClassStrings* strings, + RegExpFlags flags, Zone* zone) { + DCHECK(set.hasStrings()); + DCHECK(IsUnicodeSets(flags)); + DCHECK_NOT_NULL(strings); + + RegExpTextBuilder::SmallRegExpTreeVector string_storage( + ZoneAllocator{zone}); + RegExpTextBuilder string_builder(zone, &string_storage, flags); + const bool needs_case_folding = IsIgnoreCase(flags); + icu::UnicodeSetIterator iter(set); + iter.skipToStrings(); + while (iter.next()) { + const icu::UnicodeString& s = iter.getString(); + const char16_t* p = s.getBuffer(); + int32_t length = s.length(); + ZoneList* string = + zone->template New>(length, zone); + for (int32_t i = 0; i < length;) { + UChar32 c; + U16_NEXT(p, i, length, c); + string_builder.AddUnicodeCharacter(c); + if (needs_case_folding) { + c = u_foldCase(c, U_FOLD_CASE_DEFAULT); + } + string->Add(c, zone); + } + strings->emplace(string->ToVector(), string_builder.ToRegExp()); + string_storage.clear(); + } +} + bool LookupPropertyValueName(UProperty property, const char* property_value_name, bool negate, - ZoneList* result, Zone* zone) { + ZoneList* result_ranges, + CharacterClassStrings* result_strings, + RegExpFlags flags, Zone* zone) { UProperty property_for_lookup = property; if (property_for_lookup == UCHAR_SCRIPT_EXTENSIONS) { // For the property Script_Extensions, we have to do the property value // name lookup as if the property is Script. property_for_lookup = UCHAR_SCRIPT; @@ -1469,14 +1909,19 @@ icu::UnicodeSet set; set.applyIntPropertyValue(property, property_value, ec); bool success = ec == U_ZERO_ERROR && !set.isEmpty(); if (success) { + if (set.hasStrings()) { + ExtractStringsFromUnicodeSet(set, result_strings, flags, zone); + } + const bool needs_case_folding = IsUnicodeSets(flags) && IsIgnoreCase(flags); + if (needs_case_folding) CharacterRange::UnicodeSimpleCloseOver(set); set.removeAllStrings(); if (negate) set.complement(); for (int i = 0; i < set.getRangeCount(); i++) { - result->Add( + result_ranges->Add( CharacterRange::Range(set.getRangeStart(i), set.getRangeEnd(i)), zone); } } return success; @@ -1487,11 +1932,12 @@ return strncmp(name, literal, N + 1) == 0; } bool LookupSpecialPropertyValueName(const char* name, ZoneList* result, - bool negate, Zone* zone) { + bool negate, RegExpFlags flags, + Zone* zone) { if (NameEquals(name, "Any")) { if (negate) { // Leave the list of character ranges empty, since the negation of 'Any' // is the empty set. } else { @@ -1501,20 +1947,20 @@ result->Add(negate ? CharacterRange::Range(0x80, String::kMaxCodePoint) : CharacterRange::Range(0x0, 0x7F), zone); } else if (NameEquals(name, "Assigned")) { return LookupPropertyValueName(UCHAR_GENERAL_CATEGORY, "Unassigned", - !negate, result, zone); + !negate, result, nullptr, flags, zone); } else { return false; } return true; } // Explicitly allowlist supported binary properties. The spec forbids supporting // properties outside of this set to ensure interoperability. -bool IsSupportedBinaryProperty(UProperty property) { +bool IsSupportedBinaryProperty(UProperty property, bool unicode_sets) { switch (property) { case UCHAR_ALPHABETIC: // 'Any' is not supported by ICU. See LookupSpecialPropertyValueName. // 'ASCII' is not supported by ICU. See LookupSpecialPropertyValueName. case UCHAR_ASCII_HEX_DIGIT: @@ -1566,10 +2012,34 @@ case UCHAR_VARIATION_SELECTOR: case UCHAR_WHITE_SPACE: case UCHAR_XID_CONTINUE: case UCHAR_XID_START: return true; + case UCHAR_BASIC_EMOJI: + case UCHAR_EMOJI_KEYCAP_SEQUENCE: + case UCHAR_RGI_EMOJI_MODIFIER_SEQUENCE: + case UCHAR_RGI_EMOJI_FLAG_SEQUENCE: + case UCHAR_RGI_EMOJI_TAG_SEQUENCE: + case UCHAR_RGI_EMOJI_ZWJ_SEQUENCE: + case UCHAR_RGI_EMOJI: + return unicode_sets; + default: + break; + } + return false; +} + +bool IsBinaryPropertyOfStrings(UProperty property) { + switch (property) { + case UCHAR_BASIC_EMOJI: + case UCHAR_EMOJI_KEYCAP_SEQUENCE: + case UCHAR_RGI_EMOJI_MODIFIER_SEQUENCE: + case UCHAR_RGI_EMOJI_FLAG_SEQUENCE: + case UCHAR_RGI_EMOJI_TAG_SEQUENCE: + case UCHAR_RGI_EMOJI_ZWJ_SEQUENCE: + case UCHAR_RGI_EMOJI: + return true; default: break; } return false; } @@ -1628,28 +2098,37 @@ return true; } template bool RegExpParserImpl::AddPropertyClassRange( - ZoneList* add_to, bool negate, + ZoneList* add_to_ranges, + CharacterClassStrings* add_to_strings, bool negate, const ZoneVector& name_1, const ZoneVector& name_2) { if (name_2.empty()) { // First attempt to interpret as general category property value name. const char* name = name_1.data(); if (LookupPropertyValueName(UCHAR_GENERAL_CATEGORY_MASK, name, negate, - add_to, zone())) { + add_to_ranges, add_to_strings, flags(), + zone())) { return true; } // Interpret "Any", "ASCII", and "Assigned". - if (LookupSpecialPropertyValueName(name, add_to, negate, zone())) { + if (LookupSpecialPropertyValueName(name, add_to_ranges, negate, flags(), + zone())) { return true; } // Then attempt to interpret as binary property name with value name 'Y'. UProperty property = u_getPropertyEnum(name); - if (!IsSupportedBinaryProperty(property)) return false; + if (!IsSupportedBinaryProperty(property, unicode_sets())) return false; if (!IsExactPropertyAlias(name, property)) return false; - return LookupPropertyValueName(property, negate ? "N" : "Y", false, add_to, + // Negation of properties with strings is not allowed. + // TODO(v8:11935): Change permalink once proposal is in stage 4. + // See + // https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#sec-static-semantics-maycontainstrings + if (negate && IsBinaryPropertyOfStrings(property)) return false; + return LookupPropertyValueName(property, negate ? "N" : "Y", false, + add_to_ranges, add_to_strings, flags(), zone()); } else { // Both property name and value name are specified. Attempt to interpret // the property name as enumerated property. const char* property_name = name_1.data(); @@ -1661,12 +2140,12 @@ property = UCHAR_GENERAL_CATEGORY_MASK; } else if (property != UCHAR_SCRIPT && property != UCHAR_SCRIPT_EXTENSIONS) { return false; } - return LookupPropertyValueName(property, value_name, negate, add_to, - zone()); + return LookupPropertyValueName(property, value_name, negate, add_to_ranges, + add_to_strings, flags(), zone()); } } #else // V8_INTL_SUPPORT @@ -1676,11 +2155,12 @@ return false; } template bool RegExpParserImpl::AddPropertyClassRange( - ZoneList* add_to, bool negate, + ZoneList* add_to_ranges, + CharacterClassStrings* add_to_strings, bool negate, const ZoneVector& name_1, const ZoneVector& name_2) { return false; } #endif // V8_INTL_SUPPORT @@ -1744,18 +2224,18 @@ Advance(2); // Control letters mapped to ASCII control characters in the range // 0x00-0x1F. return controlLetter & 0x1F; } - if (unicode()) { - // With /u, invalid escapes are not treated as identity escapes. + if (IsUnicodeMode()) { + // With /u and /v, invalid escapes are not treated as identity escapes. ReportError(RegExpError::kInvalidUnicodeEscape); return 0; } if (in_class_escape_state == InClassEscapeState::kInClass) { // Inside a character class, we also accept digits and underscore as - // control characters, unless with /u. See Annex B: + // control characters, unless with /u or /v. See Annex B: // ES#prod-annexB-ClassControlLetter if ((controlLetter >= '0' && controlLetter <= '9') || controlLetter == '_') { Advance(2); return controlLetter & 0x1F; @@ -1784,24 +2264,25 @@ case '7': // For compatibility, we interpret a decimal escape that isn't // a back reference (and therefore either \0 or not valid according // to the specification) as a 1..3 digit octal character code. // ES#prod-annexB-LegacyOctalEscapeSequence - if (unicode()) { - // With /u, decimal escape is not interpreted as octal character code. + if (IsUnicodeMode()) { + // With /u or /v, decimal escape is not interpreted as octal character + // code. ReportError(RegExpError::kInvalidClassEscape); return 0; } return ParseOctalLiteral(); // CharacterEscape :: // HexEscapeSequence case 'x': { Advance(); base::uc32 value; if (ParseHexEscape(2, &value)) return value; - if (unicode()) { - // With /u, invalid escapes are not treated as identity escapes. + if (IsUnicodeMode()) { + // With /u or /v, invalid escapes are not treated as identity escapes. ReportError(RegExpError::kInvalidEscape); return 0; } // If \x is not followed by a two-digit hexadecimal, treat it // as an identity escape. @@ -1814,12 +2295,12 @@ base::uc32 value; if (ParseUnicodeEscape(&value)) { *is_escaped_unicode_character = true; return value; } - if (unicode()) { - // With /u, invalid escapes are not treated as identity escapes. + if (IsUnicodeMode()) { + // With /u or /v, invalid escapes are not treated as identity escapes. ReportError(RegExpError::kInvalidUnicodeEscape); return 0; } // If \u is not followed by a two-digit hexadecimal, treat it // as an identity escape. @@ -1832,23 +2313,31 @@ // CharacterEscape :: // IdentityEscape[?UnicodeMode, ?N] // // * With /u, no identity escapes except for syntax characters are // allowed. - // * Without /u: + // * With /v, no identity escapes except for syntax characters and + // ClassSetReservedPunctuators (if within a class) are allowed. + // * Without /u or /v: // * '\c' is not an IdentityEscape. // * '\k' is not an IdentityEscape when named captures exist. // * Otherwise, all identity escapes are allowed. - if (unicode()) { + if (unicode_sets() && in_class_escape_state == InClassEscapeState::kInClass) { + if (IsClassSetReservedPunctuator(c)) { + Advance(); + return c; + } + } + if (IsUnicodeMode()) { if (!IsSyntaxCharacterOrSlash(c)) { ReportError(RegExpError::kInvalidEscape); return 0; } Advance(); return c; } - DCHECK(!unicode()); + DCHECK(!IsUnicodeMode()); if (c == 'c') { ReportError(RegExpError::kInvalidEscape); return 0; } Advance(); @@ -1859,10 +2348,57 @@ return 0; } return c; } +// TODO(v8:11935): Change permalink once proposal is in stage 4. +// https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#prod-ClassRanges +template +RegExpTree* RegExpParserImpl::ParseClassRanges( + ZoneList* ranges, bool add_unicode_case_equivalents) { + base::uc32 char_1, char_2; + bool is_class_1, is_class_2; + while (has_more() && current() != ']') { + ParseClassEscape(ranges, zone(), add_unicode_case_equivalents, &char_1, + &is_class_1 CHECK_FAILED); + // ClassAtom + if (current() == '-') { + Advance(); + if (!has_more()) { + // If we reach the end we break out of the loop and let the + // following code report an error. + break; + } else if (current() == ']') { + if (!is_class_1) ranges->Add(CharacterRange::Singleton(char_1), zone()); + ranges->Add(CharacterRange::Singleton('-'), zone()); + break; + } + ParseClassEscape(ranges, zone(), add_unicode_case_equivalents, &char_2, + &is_class_2 CHECK_FAILED); + if (is_class_1 || is_class_2) { + // Either end is an escaped character class. Treat the '-' verbatim. + if (IsUnicodeMode()) { + // ES2015 21.2.2.15.1 step 1. + return ReportError(RegExpError::kInvalidCharacterClass); + } + if (!is_class_1) ranges->Add(CharacterRange::Singleton(char_1), zone()); + ranges->Add(CharacterRange::Singleton('-'), zone()); + if (!is_class_2) ranges->Add(CharacterRange::Singleton(char_2), zone()); + continue; + } + // ES2015 21.2.2.15.1 step 6. + if (char_1 > char_2) { + return ReportError(RegExpError::kOutOfOrderCharacterClass); + } + ranges->Add(CharacterRange::Range(char_1, char_2), zone()); + } else { + if (!is_class_1) ranges->Add(CharacterRange::Singleton(char_1), zone()); + } + } + return nullptr; +} + // https://tc39.es/ecma262/#prod-ClassEscape template void RegExpParserImpl::ParseClassEscape( ZoneList* ranges, Zone* zone, bool add_unicode_case_equivalents, base::uc32* char_out, @@ -1881,11 +2417,11 @@ case 'b': *char_out = '\b'; Advance(2); return; case '-': - if (unicode()) { + if (IsUnicodeMode()) { *char_out = next; Advance(2); return; } break; @@ -1896,24 +2432,25 @@ break; } static constexpr InClassEscapeState kInClassEscape = InClassEscapeState::kInClass; - *is_class_escape = TryParseCharacterClassEscape( - next, kInClassEscape, ranges, zone, add_unicode_case_equivalents); + *is_class_escape = + TryParseCharacterClassEscape(next, kInClassEscape, ranges, nullptr, zone, + add_unicode_case_equivalents); if (*is_class_escape) return; bool dummy = false; // Unused. *char_out = ParseCharacterEscape(kInClassEscape, &dummy); } // https://tc39.es/ecma262/#prod-CharacterClassEscape template bool RegExpParserImpl::TryParseCharacterClassEscape( base::uc32 next, InClassEscapeState in_class_escape_state, - ZoneList* ranges, Zone* zone, - bool add_unicode_case_equivalents) { + ZoneList* ranges, CharacterClassStrings* strings, + Zone* zone, bool add_unicode_case_equivalents) { DCHECK_EQ(current(), '\\'); DCHECK_EQ(Next(), next); switch (next) { case 'd': @@ -1927,17 +2464,17 @@ zone); Advance(2); return true; case 'p': case 'P': { - if (!unicode()) return false; + if (!IsUnicodeMode()) return false; bool negate = next == 'P'; Advance(2); ZoneVector name_1(zone); ZoneVector name_2(zone); if (!ParsePropertyClassName(&name_1, &name_2) || - !AddPropertyClassRange(ranges, negate, name_1, name_2)) { + !AddPropertyClassRange(ranges, strings, negate, name_1, name_2)) { ReportError(in_class_escape_state == InClassEscapeState::kInClass ? RegExpError::kInvalidClassPropertyName : RegExpError::kInvalidPropertyName); } return true; @@ -1945,10 +2482,393 @@ default: return false; } } +namespace { + +// Add |string| to |ranges| if length of |string| == 1, otherwise add |string| +// to |strings|. +void AddClassString(ZoneList* normalized_string, + RegExpTree* regexp_string, ZoneList* ranges, + CharacterClassStrings* strings, Zone* zone) { + if (normalized_string->length() == 1) { + ranges->Add(CharacterRange::Singleton(normalized_string->at(0)), zone); + } else { + strings->emplace(normalized_string->ToVector(), regexp_string); + } +} + +} // namespace + +// TODO(v8:11935): Change permalink once proposal is in stage 4. +// https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#prod-ClassStringDisjunction +template +RegExpTree* RegExpParserImpl::ParseClassStringDisjunction( + ZoneList* ranges, CharacterClassStrings* strings) { + DCHECK(unicode_sets()); + DCHECK_EQ(current(), '\\'); + DCHECK_EQ(Next(), 'q'); + Advance(2); + if (current() != '{') { + // Identity escape of 'q' is not allowed in unicode mode. + return ReportError(RegExpError::kInvalidEscape); + } + Advance(); + + ZoneList* string = + zone()->template New>(4, zone()); + RegExpTextBuilder::SmallRegExpTreeVector string_storage( + ZoneAllocator{zone()}); + RegExpTextBuilder string_builder(zone(), &string_storage, flags()); + + while (has_more() && current() != '}') { + if (current() == '|') { + AddClassString(string, string_builder.ToRegExp(), ranges, strings, + zone()); + string = zone()->template New>(4, zone()); + string_storage.clear(); + Advance(); + } else { + base::uc32 c = ParseClassSetCharacter(CHECK_FAILED); + if (ignore_case()) { +#ifdef V8_INTL_SUPPORT + c = u_foldCase(c, U_FOLD_CASE_DEFAULT); +#else + c = AsciiAlphaToLower(c); +#endif + } + string->Add(c, zone()); + string_builder.AddUnicodeCharacter(c); + } + } + + AddClassString(string, string_builder.ToRegExp(), ranges, strings, zone()); + + // We don't need to handle missing closing '}' here. + // If the character class is correctly closed, ParseClassSetCharacter will + // report an error. + DCHECK_EQ(current(), '}'); + Advance(); + return nullptr; +} + +// TODO(v8:11935): Change permalink once proposal is in stage 4. +// https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#prod-ClassSetOperand +// Tree returned based on type_out: +// * kNestedClass: RegExpClassSetExpression +// * For all other types: RegExpClassSetOperand +template +RegExpTree* RegExpParserImpl::ParseClassSetOperand( + const RegExpBuilder* builder, ClassSetOperandType* type_out) { + ZoneList* ranges = + zone()->template New>(1, zone()); + CharacterClassStrings* strings = + zone()->template New(zone()); + RegExpTree* tree = + ParseClassSetOperand(builder, type_out, ranges, strings CHECK_FAILED); + DCHECK_IMPLIES(*type_out != ClassSetOperandType::kNestedClass, + tree == nullptr); + DCHECK_IMPLIES(*type_out == ClassSetOperandType::kClassSetCharacter, + ranges->length() == 1); + DCHECK_IMPLIES(*type_out == ClassSetOperandType::kClassSetCharacter, + strings->empty()); + DCHECK_IMPLIES(*type_out == ClassSetOperandType::kNestedClass, + ranges->is_empty()); + DCHECK_IMPLIES(*type_out == ClassSetOperandType::kNestedClass, + strings->empty()); + DCHECK_IMPLIES(*type_out == ClassSetOperandType::kNestedClass, + tree->IsClassSetExpression()); + // ClassSetRange is only used within ClassSetUnion(). + DCHECK_NE(*type_out, ClassSetOperandType::kClassSetRange); + // There are no restrictions for kCharacterClassEscape. + // CharacterClassEscape includes \p{}, which can contain ranges, strings or + // both and \P{}, which could contain nothing (i.e. \P{Any}). + if (tree == nullptr) { + tree = zone()->template New(ranges, strings); + } + return tree; +} + +// TODO(v8:11935): Change permalink once proposal is in stage 4. +// https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#prod-ClassSetOperand +// Based on |type_out| either a tree is returned or ranges/strings modified. +// If a tree is returned, ranges/strings are not modified. +// If |type_out| is kNestedClass, a tree of type RegExpClassSetExpression is +// returned. For all other types, ranges is modified and nullptr is returned. +template +RegExpTree* RegExpParserImpl::ParseClassSetOperand( + const RegExpBuilder* builder, ClassSetOperandType* type_out, + ZoneList* ranges, CharacterClassStrings* strings) { + DCHECK(unicode_sets()); + base::uc32 c = current(); + if (c == '\\') { + const base::uc32 next = Next(); + if (next == 'q') { + *type_out = ClassSetOperandType::kClassStringDisjunction; + ParseClassStringDisjunction(ranges, strings CHECK_FAILED); + return nullptr; + } + static constexpr InClassEscapeState kInClassEscape = + InClassEscapeState::kInClass; + const bool add_unicode_case_equivalents = ignore_case(); + if (TryParseCharacterClassEscape(next, kInClassEscape, ranges, strings, + zone(), add_unicode_case_equivalents)) { + *type_out = ClassSetOperandType::kCharacterClassEscape; + return nullptr; + } + } + + if (c == '[') { + *type_out = ClassSetOperandType::kNestedClass; + return ParseCharacterClass(builder); + } + + *type_out = ClassSetOperandType::kClassSetCharacter; + c = ParseClassSetCharacter(CHECK_FAILED); + ranges->Add(CharacterRange::Singleton(c), zone()); + return nullptr; +} + +template +base::uc32 RegExpParserImpl::ParseClassSetCharacter() { + DCHECK(unicode_sets()); + const base::uc32 c = current(); + if (c == '\\') { + const base::uc32 next = Next(); + switch (next) { + case 'b': + Advance(2); + return '\b'; + case kEndMarker: + ReportError(RegExpError::kEscapeAtEndOfPattern); + return 0; + } + static constexpr InClassEscapeState kInClassEscape = + InClassEscapeState::kInClass; + + bool dummy = false; // Unused. + return ParseCharacterEscape(kInClassEscape, &dummy); + } + if (IsClassSetSyntaxCharacter(c)) { + ReportError(RegExpError::kInvalidCharacterInClass); + return 0; + } + if (IsClassSetReservedDoublePunctuator(c)) { + ReportError(RegExpError::kInvalidClassSetOperation); + return 0; + } + Advance(); + return c; +} + +namespace { + +bool MayContainStrings(ClassSetOperandType type, RegExpTree* operand) { + switch (type) { + case ClassSetOperandType::kClassSetCharacter: + case ClassSetOperandType::kClassSetRange: + return false; + case ClassSetOperandType::kCharacterClassEscape: + case ClassSetOperandType::kClassStringDisjunction: + return operand->AsClassSetOperand()->has_strings(); + case ClassSetOperandType::kNestedClass: + if (operand->IsClassRanges()) return false; + return operand->AsClassSetExpression()->may_contain_strings(); + } +} + +} // namespace + +// TODO(v8:11935): Change permalink once proposal is in stage 4. +// https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#prod-ClassUnion +template +RegExpTree* RegExpParserImpl::ParseClassUnion( + const RegExpBuilder* builder, bool is_negated, RegExpTree* first_operand, + ClassSetOperandType first_operand_type, ZoneList* ranges, + CharacterClassStrings* strings) { + DCHECK(unicode_sets()); + ZoneList* operands = + zone()->template New>(2, zone()); + bool may_contain_strings = false; + // Add the lhs to operands if necessary. + // Either the lhs values were added to |ranges|/|strings| (in which case + // |first_operand| is nullptr), or the lhs was evaluated to a tree and passed + // as |first_operand| (in which case |ranges| and |strings| are empty). + if (first_operand != nullptr) { + may_contain_strings = MayContainStrings(first_operand_type, first_operand); + operands->Add(first_operand, zone()); + } + ClassSetOperandType last_type = first_operand_type; + const bool needs_case_folding = ignore_case(); + while (has_more() && current() != ']') { + if (current() == '-') { + // Mix of ClassSetRange and ClassSubtraction is not allowed. + if (Next() == '-') { + return ReportError(RegExpError::kInvalidClassSetOperation); + } + Advance(); + if (!has_more()) { + // If we reach the end we break out of the loop and let the + // following code report an error. + break; + } + // If the lhs and rhs around '-' are both ClassSetCharacters, they + // represent a character range. + // In case one of them is not a ClassSetCharacter, it is a syntax error, + // as '-' can not be used unescaped within a class with /v. + // TODO(v8:11935): Change permalink once proposal is in stage 4. + // See + // https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#prod-ClassSetRange + if (last_type != ClassSetOperandType::kClassSetCharacter) { + return ReportError(RegExpError::kInvalidCharacterClass); + } + ParseClassSetOperand(builder, &last_type, ranges, strings CHECK_FAILED); + if (last_type != ClassSetOperandType::kClassSetCharacter) { + return ReportError(RegExpError::kInvalidCharacterClass); + } + // Remove the last two singleton characters added to ranges, and combine + // them into a range. + auto rhs_ranges = ranges->RemoveLast(); + auto lhs_ranges = ranges->RemoveLast(); + DCHECK(lhs_ranges.IsSingleton()); + DCHECK(rhs_ranges.IsSingleton()); + base::uc32 from = lhs_ranges.from(); + base::uc32 to = rhs_ranges.from(); + if (from > to) { + return ReportError(RegExpError::kOutOfOrderCharacterClass); + } + ranges->Add(CharacterRange::Range(from, to), zone()); + last_type = ClassSetOperandType::kClassSetRange; + } else { + DCHECK_NE(current(), '-'); + RegExpTree* operand = ParseClassSetOperand(builder, &last_type, ranges, + strings CHECK_FAILED); + if (operand != nullptr) { + may_contain_strings |= MayContainStrings(last_type, operand); + // Add the range we started building as operand and reset the current + // range. + if (!ranges->is_empty() || !strings->empty()) { + if (needs_case_folding) { + CharacterRange::AddUnicodeCaseEquivalents(ranges, zone()); + } + may_contain_strings |= !strings->empty(); + operands->Add( + zone()->template New(ranges, strings), + zone()); + ranges = zone()->template New>(2, zone()); + strings = zone()->template New(zone()); + } + operands->Add(operand, zone()); + } + } + } + + if (!has_more()) { + return ReportError(RegExpError::kUnterminatedCharacterClass); + } + + // Add the range we started building as operand. + if (!ranges->is_empty() || !strings->empty()) { + if (needs_case_folding) { + CharacterRange::AddUnicodeCaseEquivalents(ranges, zone()); + } + may_contain_strings |= !strings->empty(); + operands->Add(zone()->template New(ranges, strings), + zone()); + } + + DCHECK_EQ(current(), ']'); + Advance(); + + if (is_negated && may_contain_strings) { + return ReportError(RegExpError::kNegatedCharacterClassWithStrings); + } + + return zone()->template New( + RegExpClassSetExpression::OperationType::kUnion, is_negated, + may_contain_strings, operands); +} + +// TODO(v8:11935): Change permalink once proposal is in stage 4. +// https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#prod-ClassIntersection +template +RegExpTree* RegExpParserImpl::ParseClassIntersection( + const RegExpBuilder* builder, bool is_negated, RegExpTree* first_operand, + ClassSetOperandType first_operand_type) { + DCHECK(unicode_sets()); + DCHECK(current() == '&' && Next() == '&'); + bool may_contain_strings = + MayContainStrings(first_operand_type, first_operand); + ZoneList* operands = + zone()->template New>(2, zone()); + operands->Add(first_operand, zone()); + while (has_more() && current() != ']') { + if (current() != '&' || Next() != '&') { + return ReportError(RegExpError::kInvalidClassSetOperation); + } + Advance(2); + // [lookahead ≠ &] + if (current() == '&') { + return ReportError(RegExpError::kInvalidCharacterInClass); + } + + ClassSetOperandType operand_type; + RegExpTree* operand = + ParseClassSetOperand(builder, &operand_type CHECK_FAILED); + may_contain_strings &= MayContainStrings(operand_type, operand); + operands->Add(operand, zone()); + } + if (!has_more()) { + return ReportError(RegExpError::kUnterminatedCharacterClass); + } + if (is_negated && may_contain_strings) { + return ReportError(RegExpError::kNegatedCharacterClassWithStrings); + } + DCHECK_EQ(current(), ']'); + Advance(); + return zone()->template New( + RegExpClassSetExpression::OperationType::kIntersection, is_negated, + may_contain_strings, operands); +} + +// TODO(v8:11935): Change permalink once proposal is in stage 4. +// https://arai-a.github.io/ecma262-compare/snapshot.html?pr=2418#prod-ClassSubtraction +template +RegExpTree* RegExpParserImpl::ParseClassSubtraction( + const RegExpBuilder* builder, bool is_negated, RegExpTree* first_operand, + ClassSetOperandType first_operand_type) { + DCHECK(unicode_sets()); + DCHECK(current() == '-' && Next() == '-'); + const bool may_contain_strings = + MayContainStrings(first_operand_type, first_operand); + if (is_negated && may_contain_strings) { + return ReportError(RegExpError::kNegatedCharacterClassWithStrings); + } + ZoneList* operands = + zone()->template New>(2, zone()); + operands->Add(first_operand, zone()); + while (has_more() && current() != ']') { + if (current() != '-' || Next() != '-') { + return ReportError(RegExpError::kInvalidClassSetOperation); + } + Advance(2); + ClassSetOperandType dummy; // unused + RegExpTree* operand = ParseClassSetOperand(builder, &dummy CHECK_FAILED); + operands->Add(operand, zone()); + } + if (!has_more()) { + return ReportError(RegExpError::kUnterminatedCharacterClass); + } + DCHECK_EQ(current(), ']'); + Advance(); + return zone()->template New( + RegExpClassSetExpression::OperationType::kSubtraction, is_negated, + may_contain_strings, operands); +} + +// https://tc39.es/ecma262/#prod-CharacterClass template RegExpTree* RegExpParserImpl::ParseCharacterClass( const RegExpBuilder* builder) { DCHECK_EQ(current(), '['); Advance(); @@ -1957,57 +2877,61 @@ is_negated = true; Advance(); } ZoneList* ranges = zone()->template New>(2, zone()); - bool add_unicode_case_equivalents = unicode() && builder->ignore_case(); - while (has_more() && current() != ']') { - base::uc32 char_1, char_2; - bool is_class_1, is_class_2; - ParseClassEscape(ranges, zone(), add_unicode_case_equivalents, &char_1, - &is_class_1 CHECK_FAILED); - if (current() == '-') { - Advance(); - if (current() == kEndMarker) { - // If we reach the end we break out of the loop and let the - // following code report an error. - break; - } else if (current() == ']') { - if (!is_class_1) ranges->Add(CharacterRange::Singleton(char_1), zone()); - ranges->Add(CharacterRange::Singleton('-'), zone()); + if (current() == ']') { + Advance(); + RegExpClassRanges::ClassRangesFlags class_ranges_flags; + if (is_negated) class_ranges_flags = RegExpClassRanges::NEGATED; + return zone()->template New(zone(), ranges, + class_ranges_flags); + } + + if (!unicode_sets()) { + bool add_unicode_case_equivalents = IsUnicodeMode() && ignore_case(); + ParseClassRanges(ranges, add_unicode_case_equivalents CHECK_FAILED); + if (!has_more()) { + return ReportError(RegExpError::kUnterminatedCharacterClass); + } + DCHECK_EQ(current(), ']'); + Advance(); + RegExpClassRanges::ClassRangesFlags character_class_flags; + if (is_negated) character_class_flags = RegExpClassRanges::NEGATED; + return zone()->template New(zone(), ranges, + character_class_flags); + } else { + ClassSetOperandType operand_type; + CharacterClassStrings* strings = + zone()->template New(zone()); + RegExpTree* operand = ParseClassSetOperand(builder, &operand_type, ranges, + strings CHECK_FAILED); + switch (current()) { + case '-': + if (Next() == '-') { + if (operand == nullptr) { + operand = + zone()->template New(ranges, strings); + } + return ParseClassSubtraction(builder, is_negated, operand, + operand_type); + } + // ClassSetRange is handled in ParseClassUnion(). break; - } - ParseClassEscape(ranges, zone(), add_unicode_case_equivalents, &char_2, - &is_class_2 CHECK_FAILED); - if (is_class_1 || is_class_2) { - // Either end is an escaped character class. Treat the '-' verbatim. - if (unicode()) { - // ES2015 21.2.2.15.1 step 1. - return ReportError(RegExpError::kInvalidCharacterClass); + case '&': + if (Next() == '&') { + if (operand == nullptr) { + operand = + zone()->template New(ranges, strings); + } + return ParseClassIntersection(builder, is_negated, operand, + operand_type); } - if (!is_class_1) ranges->Add(CharacterRange::Singleton(char_1), zone()); - ranges->Add(CharacterRange::Singleton('-'), zone()); - if (!is_class_2) ranges->Add(CharacterRange::Singleton(char_2), zone()); - continue; - } - // ES2015 21.2.2.15.1 step 6. - if (char_1 > char_2) { - return ReportError(RegExpError::kOutOfOrderCharacterClass); - } - ranges->Add(CharacterRange::Range(char_1, char_2), zone()); - } else { - if (!is_class_1) ranges->Add(CharacterRange::Singleton(char_1), zone()); } - } - if (!has_more()) { - return ReportError(RegExpError::kUnterminatedCharacterClass); + return ParseClassUnion(builder, is_negated, operand, operand_type, ranges, + strings); } - Advance(); - RegExpCharacterClass::CharacterClassFlags character_class_flags; - if (is_negated) character_class_flags = RegExpCharacterClass::NEGATED; - return zone()->template New(zone(), ranges, - character_class_flags); } #undef CHECK_FAILED template @@ -2023,11 +2947,11 @@ return false; } DCHECK_NOT_NULL(tree); DCHECK_EQ(error_, RegExpError::kNone); - if (FLAG_trace_regexp_parser) { + if (v8_flags.trace_regexp_parser) { StdoutStream os; tree->Print(os, zone()); os << "\n"; } @@ -2038,156 +2962,66 @@ result->capture_count = capture_count; result->named_captures = GetNamedCaptures(); return true; } -void RegExpBuilder::AddLeadSurrogate(base::uc16 lead_surrogate) { - DCHECK(unibrow::Utf16::IsLeadSurrogate(lead_surrogate)); - FlushPendingSurrogate(); - // Hold onto the lead surrogate, waiting for a trail surrogate to follow. - pending_surrogate_ = lead_surrogate; -} - -void RegExpBuilder::AddTrailSurrogate(base::uc16 trail_surrogate) { - DCHECK(unibrow::Utf16::IsTrailSurrogate(trail_surrogate)); - if (pending_surrogate_ != kNoPendingSurrogate) { - base::uc16 lead_surrogate = pending_surrogate_; - pending_surrogate_ = kNoPendingSurrogate; - DCHECK(unibrow::Utf16::IsLeadSurrogate(lead_surrogate)); - base::uc32 combined = - unibrow::Utf16::CombineSurrogatePair(lead_surrogate, trail_surrogate); - if (NeedsDesugaringForIgnoreCase(combined)) { - AddCharacterClassForDesugaring(combined); - } else { - ZoneList surrogate_pair(2, zone()); - surrogate_pair.Add(lead_surrogate, zone()); - surrogate_pair.Add(trail_surrogate, zone()); - RegExpAtom* atom = - zone()->New(surrogate_pair.ToConstVector()); - AddAtom(atom); - } - } else { - pending_surrogate_ = trail_surrogate; - FlushPendingSurrogate(); - } -} - -void RegExpBuilder::FlushPendingSurrogate() { - if (pending_surrogate_ != kNoPendingSurrogate) { - DCHECK(unicode()); - base::uc32 c = pending_surrogate_; - pending_surrogate_ = kNoPendingSurrogate; - AddCharacterClassForDesugaring(c); - } -} - -void RegExpBuilder::FlushCharacters() { - FlushPendingSurrogate(); - pending_empty_ = false; - if (characters_ != nullptr) { - RegExpTree* atom = zone()->New(characters_->ToConstVector()); - characters_ = nullptr; - text_.emplace_back(atom); - LAST(ADD_ATOM); - } -} - -void RegExpBuilder::FlushText() { - FlushCharacters(); - size_t num_text = text_.size(); - if (num_text == 0) { - return; - } else if (num_text == 1) { - terms_.emplace_back(text_.back()); - } else { - RegExpText* text = zone()->New(zone()); - for (size_t i = 0; i < num_text; i++) { - text_[i]->AppendToText(text, zone()); - } - terms_.emplace_back(text); - } - text_.clear(); -} +void RegExpBuilder::FlushText() { text_builder().FlushText(); } void RegExpBuilder::AddCharacter(base::uc16 c) { - FlushPendingSurrogate(); pending_empty_ = false; - if (NeedsDesugaringForIgnoreCase(c)) { - AddCharacterClassForDesugaring(c); - } else { - if (characters_ == nullptr) { - characters_ = zone()->New>(4, zone()); - } - characters_->Add(c, zone()); - LAST(ADD_CHAR); - } + text_builder().AddCharacter(c); } void RegExpBuilder::AddUnicodeCharacter(base::uc32 c) { - if (c > static_cast(unibrow::Utf16::kMaxNonSurrogateCharCode)) { - DCHECK(unicode()); - AddLeadSurrogate(unibrow::Utf16::LeadSurrogate(c)); - AddTrailSurrogate(unibrow::Utf16::TrailSurrogate(c)); - } else if (unicode() && unibrow::Utf16::IsLeadSurrogate(c)) { - AddLeadSurrogate(c); - } else if (unicode() && unibrow::Utf16::IsTrailSurrogate(c)) { - AddTrailSurrogate(c); - } else { - AddCharacter(static_cast(c)); - } + pending_empty_ = false; + text_builder().AddUnicodeCharacter(c); } void RegExpBuilder::AddEscapedUnicodeCharacter(base::uc32 character) { - // A lead or trail surrogate parsed via escape sequence will not - // pair up with any preceding lead or following trail surrogate. - FlushPendingSurrogate(); - AddUnicodeCharacter(character); - FlushPendingSurrogate(); + pending_empty_ = false; + text_builder().AddEscapedUnicodeCharacter(character); } -void RegExpBuilder::AddEmpty() { pending_empty_ = true; } - -void RegExpBuilder::AddCharacterClass(RegExpCharacterClass* cc) { - if (NeedsDesugaringForUnicode(cc)) { - // With /u, character class needs to be desugared, so it - // must be a standalone term instead of being part of a RegExpText. - AddTerm(cc); - } else { - AddAtom(cc); - } +void RegExpBuilder::AddEmpty() { + text_builder().FlushPendingSurrogate(); + pending_empty_ = true; } -void RegExpBuilder::AddCharacterClassForDesugaring(base::uc32 c) { - AddTerm(zone()->New( - zone(), CharacterRange::List(zone(), CharacterRange::Singleton(c)))); +void RegExpBuilder::AddClassRanges(RegExpClassRanges* cc) { + pending_empty_ = false; + text_builder().AddClassRanges(cc); } void RegExpBuilder::AddAtom(RegExpTree* term) { if (term->IsEmpty()) { AddEmpty(); return; } + pending_empty_ = false; if (term->IsTextElement()) { - FlushCharacters(); - text_.emplace_back(term); + text_builder().AddAtom(term); } else { FlushText(); terms_.emplace_back(term); } - LAST(ADD_ATOM); } void RegExpBuilder::AddTerm(RegExpTree* term) { - FlushText(); - terms_.emplace_back(term); - LAST(ADD_ATOM); + DCHECK(!term->IsEmpty()); + pending_empty_ = false; + if (term->IsTextElement()) { + text_builder().AddTerm(term); + } else { + FlushText(); + terms_.emplace_back(term); + } } void RegExpBuilder::AddAssertion(RegExpTree* assert) { FlushText(); + pending_empty_ = false; terms_.emplace_back(assert); - LAST(ADD_ASSERT); } void RegExpBuilder::NewAlternative() { FlushTerms(); } void RegExpBuilder::FlushTerms() { @@ -2203,52 +3037,10 @@ zone()->New(zone()->New>( base::VectorOf(terms_.begin(), terms_.size()), zone())); } alternatives_.emplace_back(alternative); terms_.clear(); - LAST(ADD_NONE); -} - -bool RegExpBuilder::NeedsDesugaringForUnicode(RegExpCharacterClass* cc) { - if (!unicode()) return false; - // TODO(yangguo): we could be smarter than this. Case-insensitivity does not - // necessarily mean that we need to desugar. It's probably nicer to have a - // separate pass to figure out unicode desugarings. - if (ignore_case()) return true; - ZoneList* ranges = cc->ranges(zone()); - CharacterRange::Canonicalize(ranges); - - if (cc->is_negated()) { - ZoneList* negated_ranges = - zone()->New>(ranges->length(), zone()); - CharacterRange::Negate(ranges, negated_ranges, zone()); - ranges = negated_ranges; - } - - for (int i = ranges->length() - 1; i >= 0; i--) { - base::uc32 from = ranges->at(i).from(); - base::uc32 to = ranges->at(i).to(); - // Check for non-BMP characters. - if (to >= kNonBmpStart) return true; - // Check for lone surrogates. - if (from <= kTrailSurrogateEnd && to >= kLeadSurrogateStart) return true; - } - return false; -} - -bool RegExpBuilder::NeedsDesugaringForIgnoreCase(base::uc32 c) { -#ifdef V8_INTL_SUPPORT - if (unicode() && ignore_case()) { - icu::UnicodeSet set(c, c); - set.closeOver(USET_CASE_INSENSITIVE); - set.removeAllStrings(); - return set.size() > 1; - } - // In the case where ICU is not included, we act as if the unicode flag is - // not set, and do not desugar. -#endif // V8_INTL_SUPPORT - return false; } RegExpTree* RegExpBuilder::ToRegExp() { FlushTerms(); size_t num_alternatives = alternatives_.size(); @@ -2258,50 +3050,30 @@ base::VectorOf(alternatives_.begin(), alternatives_.size()), zone())); } bool RegExpBuilder::AddQuantifierToAtom( int min, int max, RegExpQuantifier::QuantifierType quantifier_type) { - FlushPendingSurrogate(); if (pending_empty_) { pending_empty_ = false; return true; } - RegExpTree* atom; - if (characters_ != nullptr) { - DCHECK(last_added_ == ADD_CHAR); - // Last atom was character. - base::Vector char_vector = characters_->ToConstVector(); - int num_chars = char_vector.length(); - if (num_chars > 1) { - base::Vector prefix = - char_vector.SubVector(0, num_chars - 1); - text_.emplace_back(zone()->New(prefix)); - char_vector = char_vector.SubVector(num_chars - 1, num_chars); - } - characters_ = nullptr; - atom = zone()->New(char_vector); - FlushText(); - } else if (text_.size() > 0) { - DCHECK(last_added_ == ADD_ATOM); - atom = text_.back(); - text_.pop_back(); + RegExpTree* atom = text_builder().PopLastAtom(); + if (atom != nullptr) { FlushText(); } else if (terms_.size() > 0) { - DCHECK(last_added_ == ADD_ATOM); atom = terms_.back(); terms_.pop_back(); if (atom->IsLookaround()) { - // With /u, lookarounds are not quantifiable. - if (unicode()) return false; + // With /u or /v, lookarounds are not quantifiable. + if (IsUnicodeMode()) return false; // Lookbehinds are not quantifiable. if (atom->AsLookaround()->type() == RegExpLookaround::LOOKBEHIND) { return false; } } if (atom->max_match() == 0) { // Guaranteed to only match an empty string. - LAST(ADD_TERM); if (min == 0) { return true; } terms_.emplace_back(atom); return true; @@ -2310,11 +3082,10 @@ // Only call immediately after adding an atom or character! UNREACHABLE(); } terms_.emplace_back( zone()->New(min, max, quantifier_type, atom)); - LAST(ADD_TERM); return true; } template class RegExpParserImpl; template class RegExpParserImpl; @@ -2359,9 +3130,7 @@ const DisallowGarbageCollection&); template bool RegExpParser::VerifyRegExpSyntax( Zone*, uintptr_t, const base::uc16*, int, RegExpFlags, RegExpCompileData*, const DisallowGarbageCollection&); -#undef LAST - } // namespace internal } // namespace v8 diff --git a/js/src/irregexp/imported/regexp.h b/js/src/irregexp/imported/regexp.h --- a/js/src/irregexp/imported/regexp.h +++ b/js/src/irregexp/imported/regexp.h @@ -65,10 +65,13 @@ class RegExp final : public AllStatic { public: // Whether the irregexp engine generates interpreter bytecode. static bool CanGenerateBytecode(); + // Verify that the given flags combination is valid. + V8_EXPORT_PRIVATE static bool VerifyFlags(RegExpFlags flags); + // Verify the given pattern, i.e. check that parsing succeeds. If // verification fails, `regexp_error_out` is set. template static bool VerifySyntax(Zone* zone, uintptr_t stack_limit, const CharT* input, int input_length, diff --git a/js/src/irregexp/imported/special-case.h b/js/src/irregexp/imported/special-case.h --- a/js/src/irregexp/imported/special-case.h +++ b/js/src/irregexp/imported/special-case.h @@ -68,15 +68,25 @@ // result of closeOver for each BMP character, and finding sets for // which at least one character has a different canonical value than // another character. Characters that match no other characters in // their equivalence class are added to IgnoreSet. Characters that // match at least one other character are added to SpecialAddSet. +// +// For unicode ignoreCase ("iu" and "iv"), +// UnicodeSet::closeOver(USET_CASE_INSENSITIVE) adds all characters that are in +// the same equivalence class. This includes characaters that are in the same +// equivalence class using full case folding. According to the spec, only +// simple case folding shall be considered. We therefore create +// UnicodeNonSimpleCloseOverSet containing all characters for which +// UnicodeSet::closeOver adds characters that are not simple case folds. This +// set should be used similar to IgnoreSet described above. class RegExpCaseFolding final : public AllStatic { public: static const icu::UnicodeSet& IgnoreSet(); static const icu::UnicodeSet& SpecialAddSet(); + static const icu::UnicodeSet& UnicodeNonSimpleCloseOverSet(); // This implements ECMAScript 2020 21.2.2.8.2 (Runtime Semantics: // Canonicalize) step 3, which is used to determine whether // characters match when ignoreCase is true and unicode is false. static UChar32 Canonicalize(UChar32 ch) { diff --git a/js/src/irregexp/moz.yaml b/js/src/irregexp/moz.yaml --- a/js/src/irregexp/moz.yaml +++ b/js/src/irregexp/moz.yaml @@ -7,12 +7,12 @@ origin: name: irregexp description: A fast regular expression engine from V8 url: https://v8.dev - release: adb5e163ac6356bf2e39891de6a46e0e49709836 - revision: adb5e163ac6356bf2e39891de6a46e0e49709836 + release: bf20040552ba0653a6bd2cf98a40ba21f7bcad31 (Wed Dec 07 16:52:46 2022). + revision: bf20040552ba0653a6bd2cf98a40ba21f7bcad31 license: BSD-3-Clause license-file: LICENSE.v8 vendoring: