--- embedaddon/pcre/pcre_compile.c 2012/10/09 09:19:17 1.1.1.3 +++ embedaddon/pcre/pcre_compile.c 2014/06/15 19:46:03 1.1.1.5 @@ -6,7 +6,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge + Copyright (c) 1997-2013 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -53,7 +53,7 @@ supporting internal functions that are not used by oth #include "pcre_internal.h" -/* When PCRE_DEBUG is defined, we need the pcre(16)_printint() function, which +/* When PCRE_DEBUG is defined, we need the pcre(16|32)_printint() function, which is also used by pcretest. PCRE_DEBUG is not defined when building a production library. We do not need to select pcre16_printint.c specially, because the COMPILE_PCREx macro will already be appropriately set. */ @@ -68,7 +68,7 @@ COMPILE_PCREx macro will already be appropriately set. /* Macro for setting individual bits in class bitmaps. */ -#define SETBIT(a,b) a[b/8] |= (1 << (b%8)) +#define SETBIT(a,b) a[(b)/8] |= (1 << ((b)&7)) /* Maximum length value to check against when making sure that the integer that holds the compiled pattern length does not overflow. We make it a bit less than @@ -77,7 +77,19 @@ to check them every time. */ #define OFLOW_MAX (INT_MAX - 20) +/* Definitions to allow mutual recursion */ +static int + add_list_to_class(pcre_uint8 *, pcre_uchar **, int, compile_data *, + const pcre_uint32 *, unsigned int); + +static BOOL + compile_regex(int, pcre_uchar **, const pcre_uchar **, int *, BOOL, BOOL, int, int, + pcre_uint32 *, pcre_int32 *, pcre_uint32 *, pcre_int32 *, branch_chain *, + compile_data *, int *); + + + /************************************************* * Code parameters and static tables * *************************************************/ @@ -103,6 +115,13 @@ kicks in at the same number of forward references in a #define COMPILE_WORK_SIZE (2048*LINK_SIZE) #define COMPILE_WORK_SIZE_MAX (100*COMPILE_WORK_SIZE) +/* This value determines the size of the initial vector that is used for +remembering named groups during the pre-compile. It is allocated on the stack, +but if it is too small, it is expanded using malloc(), in a similar way to the +workspace. The value is the number of slots in the list. */ + +#define NAMED_GROUP_LIST_SIZE 20 + /* The overrun tests check for a slightly smaller size so that they detect the overrun before it actually does run off the end of the data block. */ @@ -110,8 +129,11 @@ overrun before it actually does run off the end of the /* Private flags added to firstchar and reqchar. */ -#define REQ_CASELESS 0x10000000l /* Indicates caselessness */ -#define REQ_VARY 0x20000000l /* Reqchar followed non-literal item */ +#define REQ_CASELESS (1 << 0) /* Indicates caselessness */ +#define REQ_VARY (1 << 1) /* Reqchar followed non-literal item */ +/* Negative values for the firstchar and reqchar flags */ +#define REQ_UNSET (-2) +#define REQ_NONE (-1) /* Repeated character flags. */ @@ -238,11 +260,25 @@ static const verbitem verbs[] = { static const int verbcount = sizeof(verbs)/sizeof(verbitem); +/* Substitutes for [[:<:]] and [[:>:]], which mean start and end of word in +another regex library. */ + +static const pcre_uchar sub_start_of_word[] = { + CHAR_BACKSLASH, CHAR_b, CHAR_LEFT_PARENTHESIS, CHAR_QUESTION_MARK, + CHAR_EQUALS_SIGN, CHAR_BACKSLASH, CHAR_w, CHAR_RIGHT_PARENTHESIS, '\0' }; + +static const pcre_uchar sub_end_of_word[] = { + CHAR_BACKSLASH, CHAR_b, CHAR_LEFT_PARENTHESIS, CHAR_QUESTION_MARK, + CHAR_LESS_THAN_SIGN, CHAR_EQUALS_SIGN, CHAR_BACKSLASH, CHAR_w, + CHAR_RIGHT_PARENTHESIS, '\0' }; + + /* Tables of names of POSIX character classes and their lengths. The names are now all in a single string, to reduce the number of relocations when a shared library is dynamically loaded. The list of lengths is terminated by a zero length entry. The first three must be alpha, lower, upper, as this is assumed -for handling case independence. */ +for handling case independence. The indices for graph, print, and punct are +needed, so identify them. */ static const char posix_names[] = STRING_alpha0 STRING_lower0 STRING_upper0 STRING_alnum0 @@ -253,6 +289,11 @@ static const char posix_names[] = static const pcre_uint8 posix_name_lengths[] = { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 6, 0 }; +#define PC_GRAPH 8 +#define PC_PRINT 9 +#define PC_PUNCT 10 + + /* Table of class bit maps for each POSIX class. Each class is formed from a base map, with an optional addition or removal of another map. Then, for some classes, there is some additional tweaking: for [:blank:] the vertical space @@ -280,9 +321,8 @@ static const int posix_class_maps[] = { cbit_xdigit,-1, 0 /* xdigit */ }; -/* Table of substitutes for \d etc when PCRE_UCP is set. The POSIX class -substitutes must be in the order of the names, defined above, and there are -both positive and negative cases. NULL means no substitute. */ +/* Table of substitutes for \d etc when PCRE_UCP is set. They are replaced by +Unicode property escapes. */ #ifdef SUPPORT_UCP static const pcre_uchar string_PNd[] = { @@ -307,12 +347,18 @@ static const pcre_uchar string_pXwd[] = { static const pcre_uchar *substitutes[] = { string_PNd, /* \D */ string_pNd, /* \d */ - string_PXsp, /* \S */ /* NOTE: Xsp is Perl space */ - string_pXsp, /* \s */ + string_PXsp, /* \S */ /* Xsp is Perl space, but from 8.34, Perl */ + string_pXsp, /* \s */ /* space and POSIX space are the same. */ string_PXwd, /* \W */ string_pXwd /* \w */ }; +/* The POSIX class substitutes must be in the order of the POSIX class names, +defined above, and there are both positive and negative cases. NULL means no +general substitute of a Unicode property escape (\p or \P). However, for some +POSIX classes (e.g. graph, print, punct) a special property code is compiled +directly. */ + static const pcre_uchar string_pL[] = { CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, CHAR_L, CHAR_RIGHT_CURLY_BRACKET, '\0' }; @@ -360,8 +406,8 @@ static const pcre_uchar *posix_substitutes[] = { NULL, /* graph */ NULL, /* print */ NULL, /* punct */ - string_pXps, /* space */ /* NOTE: Xps is POSIX space */ - string_pXwd, /* word */ + string_pXps, /* space */ /* Xps is POSIX space, but from 8.34 */ + string_pXwd, /* word */ /* Perl and POSIX space are the same */ NULL, /* xdigit */ /* Negated cases */ string_PL, /* ^alpha */ @@ -375,8 +421,8 @@ static const pcre_uchar *posix_substitutes[] = { NULL, /* ^graph */ NULL, /* ^print */ NULL, /* ^punct */ - string_PXps, /* ^space */ /* NOTE: Xps is POSIX space */ - string_PXwd, /* ^word */ + string_PXps, /* ^space */ /* Xps is POSIX space, but from 8.34 */ + string_PXwd, /* ^word */ /* Perl and POSIX space are the same */ NULL /* ^xdigit */ }; #define POSIX_SUBSIZE (sizeof(posix_substitutes) / sizeof(pcre_uchar *)) @@ -440,7 +486,7 @@ static const char error_texts[] = "POSIX collating elements are not supported\0" "this version of PCRE is compiled without UTF support\0" "spare error\0" /** DEAD **/ - "character value in \\x{...} sequence is too large\0" + "character value in \\x{} or \\o{} is too large\0" /* 35 */ "invalid condition (?(0)\0" "\\C not allowed in lookbehind assertion\0" @@ -472,7 +518,7 @@ static const char error_texts[] = "a numbered reference must not be zero\0" "an argument is not allowed for (*ACCEPT), (*FAIL), or (*COMMIT)\0" /* 60 */ - "(*VERB) not recognized\0" + "(*VERB) not recognized or malformed\0" "number is too big\0" "subpattern name expected\0" "digit expected after (?+\0" @@ -492,6 +538,15 @@ static const char error_texts[] = /* 75 */ "name is too long in (*MARK), (*PRUNE), (*SKIP), or (*THEN)\0" "character value in \\u.... sequence is too large\0" + "invalid UTF-32 string\0" + "setting UTF is disabled by the application\0" + "non-hex character in \\x{} (closing brace missing?)\0" + /* 80 */ + "non-octal character in \\o{} (closing brace missing?)\0" + "missing opening brace after \\o\0" + "parentheses are too deeply nested\0" + "invalid range in character class\0" + "group name must start with a non-digit\0" ; /* Table to identify digits and hex digits. This is used when compiling @@ -631,14 +686,184 @@ static const pcre_uint8 ebcdic_chartab[] = { /* charta #endif -/* Definition to allow mutual recursion */ +/* This table is used to check whether auto-possessification is possible +between adjacent character-type opcodes. The left-hand (repeated) opcode is +used to select the row, and the right-hand opcode is use to select the column. +A value of 1 means that auto-possessification is OK. For example, the second +value in the first row means that \D+\d can be turned into \D++\d. -static BOOL - compile_regex(int, pcre_uchar **, const pcre_uchar **, int *, BOOL, BOOL, int, int, - int *, int *, branch_chain *, compile_data *, int *); +The Unicode property types (\P and \p) have to be present to fill out the table +because of what their opcode values are, but the table values should always be +zero because property types are handled separately in the code. The last four +columns apply to items that cannot be repeated, so there is no need to have +rows for them. Note that OP_DIGIT etc. are generated only when PCRE_UCP is +*not* set. When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */ +#define APTROWS (LAST_AUTOTAB_LEFT_OP - FIRST_AUTOTAB_OP + 1) +#define APTCOLS (LAST_AUTOTAB_RIGHT_OP - FIRST_AUTOTAB_OP + 1) +static const pcre_uint8 autoposstab[APTROWS][APTCOLS] = { +/* \D \d \S \s \W \w . .+ \C \P \p \R \H \h \V \v \X \Z \z $ $M */ + { 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \D */ + { 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 }, /* \d */ + { 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 }, /* \S */ + { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \s */ + { 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \W */ + { 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 }, /* \w */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* . */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* .+ */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \C */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* \P */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* \p */ + { 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0 }, /* \R */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0 }, /* \H */ + { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0 }, /* \h */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0 }, /* \V */ + { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0 }, /* \v */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 } /* \X */ +}; + +/* This table is used to check whether auto-possessification is possible +between adjacent Unicode property opcodes (OP_PROP and OP_NOTPROP). The +left-hand (repeated) opcode is used to select the row, and the right-hand +opcode is used to select the column. The values are as follows: + + 0 Always return FALSE (never auto-possessify) + 1 Character groups are distinct (possessify if both are OP_PROP) + 2 Check character categories in the same group (general or particular) + 3 TRUE if the two opcodes are not the same (PROP vs NOTPROP) + + 4 Check left general category vs right particular category + 5 Check right general category vs left particular category + + 6 Left alphanum vs right general category + 7 Left space vs right general category + 8 Left word vs right general category + + 9 Right alphanum vs left general category + 10 Right space vs left general category + 11 Right word vs left general category + + 12 Left alphanum vs right particular category + 13 Left space vs right particular category + 14 Left word vs right particular category + + 15 Right alphanum vs left particular category + 16 Right space vs left particular category + 17 Right word vs left particular category +*/ + +static const pcre_uint8 propposstab[PT_TABSIZE][PT_TABSIZE] = { +/* ANY LAMP GC PC SC ALNUM SPACE PXSPACE WORD CLIST UCNC */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* PT_ANY */ + { 0, 3, 0, 0, 0, 3, 1, 1, 0, 0, 0 }, /* PT_LAMP */ + { 0, 0, 2, 4, 0, 9, 10, 10, 11, 0, 0 }, /* PT_GC */ + { 0, 0, 5, 2, 0, 15, 16, 16, 17, 0, 0 }, /* PT_PC */ + { 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0 }, /* PT_SC */ + { 0, 3, 6, 12, 0, 3, 1, 1, 0, 0, 0 }, /* PT_ALNUM */ + { 0, 1, 7, 13, 0, 1, 3, 3, 1, 0, 0 }, /* PT_SPACE */ + { 0, 1, 7, 13, 0, 1, 3, 3, 1, 0, 0 }, /* PT_PXSPACE */ + { 0, 0, 8, 14, 0, 0, 1, 1, 3, 0, 0 }, /* PT_WORD */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* PT_CLIST */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3 } /* PT_UCNC */ +}; + +/* This table is used to check whether auto-possessification is possible +between adjacent Unicode property opcodes (OP_PROP and OP_NOTPROP) when one +specifies a general category and the other specifies a particular category. The +row is selected by the general category and the column by the particular +category. The value is 1 if the particular category is not part of the general +category. */ + +static const pcre_uint8 catposstab[7][30] = { +/* Cc Cf Cn Co Cs Ll Lm Lo Lt Lu Mc Me Mn Nd Nl No Pc Pd Pe Pf Pi Po Ps Sc Sk Sm So Zl Zp Zs */ + { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* C */ + { 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* L */ + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* M */ + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* N */ + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1 }, /* P */ + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1 }, /* S */ + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0 } /* Z */ +}; + +/* This table is used when checking ALNUM, (PX)SPACE, SPACE, and WORD against +a general or particular category. The properties in each row are those +that apply to the character set in question. Duplication means that a little +unnecessary work is done when checking, but this keeps things much simpler +because they can all use the same code. For more details see the comment where +this table is used. + +Note: SPACE and PXSPACE used to be different because Perl excluded VT from +"space", but from Perl 5.18 it's included, so both categories are treated the +same here. */ + +static const pcre_uint8 posspropstab[3][4] = { + { ucp_L, ucp_N, ucp_N, ucp_Nl }, /* ALNUM, 3rd and 4th values redundant */ + { ucp_Z, ucp_Z, ucp_C, ucp_Cc }, /* SPACE and PXSPACE, 2nd value redundant */ + { ucp_L, ucp_N, ucp_P, ucp_Po } /* WORD */ +}; + +/* This table is used when converting repeating opcodes into possessified +versions as a result of an explicit possessive quantifier such as ++. A zero +value means there is no possessified version - in those cases the item in +question must be wrapped in ONCE brackets. The table is truncated at OP_CALLOUT +because all relevant opcodes are less than that. */ + +static const pcre_uint8 opcode_possessify[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0 - 15 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 16 - 31 */ + + 0, /* NOTI */ + OP_POSSTAR, 0, /* STAR, MINSTAR */ + OP_POSPLUS, 0, /* PLUS, MINPLUS */ + OP_POSQUERY, 0, /* QUERY, MINQUERY */ + OP_POSUPTO, 0, /* UPTO, MINUPTO */ + 0, /* EXACT */ + 0, 0, 0, 0, /* POS{STAR,PLUS,QUERY,UPTO} */ + + OP_POSSTARI, 0, /* STARI, MINSTARI */ + OP_POSPLUSI, 0, /* PLUSI, MINPLUSI */ + OP_POSQUERYI, 0, /* QUERYI, MINQUERYI */ + OP_POSUPTOI, 0, /* UPTOI, MINUPTOI */ + 0, /* EXACTI */ + 0, 0, 0, 0, /* POS{STARI,PLUSI,QUERYI,UPTOI} */ + + OP_NOTPOSSTAR, 0, /* NOTSTAR, NOTMINSTAR */ + OP_NOTPOSPLUS, 0, /* NOTPLUS, NOTMINPLUS */ + OP_NOTPOSQUERY, 0, /* NOTQUERY, NOTMINQUERY */ + OP_NOTPOSUPTO, 0, /* NOTUPTO, NOTMINUPTO */ + 0, /* NOTEXACT */ + 0, 0, 0, 0, /* NOTPOS{STAR,PLUS,QUERY,UPTO} */ + + OP_NOTPOSSTARI, 0, /* NOTSTARI, NOTMINSTARI */ + OP_NOTPOSPLUSI, 0, /* NOTPLUSI, NOTMINPLUSI */ + OP_NOTPOSQUERYI, 0, /* NOTQUERYI, NOTMINQUERYI */ + OP_NOTPOSUPTOI, 0, /* NOTUPTOI, NOTMINUPTOI */ + 0, /* NOTEXACTI */ + 0, 0, 0, 0, /* NOTPOS{STARI,PLUSI,QUERYI,UPTOI} */ + + OP_TYPEPOSSTAR, 0, /* TYPESTAR, TYPEMINSTAR */ + OP_TYPEPOSPLUS, 0, /* TYPEPLUS, TYPEMINPLUS */ + OP_TYPEPOSQUERY, 0, /* TYPEQUERY, TYPEMINQUERY */ + OP_TYPEPOSUPTO, 0, /* TYPEUPTO, TYPEMINUPTO */ + 0, /* TYPEEXACT */ + 0, 0, 0, 0, /* TYPEPOS{STAR,PLUS,QUERY,UPTO} */ + + OP_CRPOSSTAR, 0, /* CRSTAR, CRMINSTAR */ + OP_CRPOSPLUS, 0, /* CRPLUS, CRMINPLUS */ + OP_CRPOSQUERY, 0, /* CRQUERY, CRMINQUERY */ + OP_CRPOSRANGE, 0, /* CRRANGE, CRMINRANGE */ + 0, 0, 0, 0, /* CRPOS{STAR,PLUS,QUERY,RANGE} */ + + 0, 0, 0, /* CLASS, NCLASS, XCLASS */ + 0, 0, /* REF, REFI */ + 0, 0, /* DNREF, DNREFI */ + 0, 0 /* RECURSE, CALLOUT */ +}; + + + /************************************************* * Find an error text * *************************************************/ @@ -658,13 +883,14 @@ find_error_text(int n) const char *s = error_texts; for (; n > 0; n--) { - while (*s++ != 0) {}; - if (*s == 0) return "Error text not found (please report)"; + while (*s++ != CHAR_NULL) {}; + if (*s == CHAR_NULL) return "Error text not found (please report)"; } return s; } + /************************************************* * Expand the workspace * *************************************************/ @@ -742,33 +968,35 @@ return (*p == CHAR_RIGHT_CURLY_BRACKET); *************************************************/ /* This function is called when a \ has been encountered. It either returns a -positive value for a simple escape such as \n, or a negative value which -encodes one of the more complicated things such as \d. A backreference to group -n is returned as -(ESC_REF + n); ESC_REF is the highest ESC_xxx macro. When -UTF-8 is enabled, a positive value greater than 255 may be returned. On entry, -ptr is pointing at the \. On exit, it is on the final character of the escape -sequence. +positive value for a simple escape such as \n, or 0 for a data character which +will be placed in chptr. A backreference to group n is returned as negative n. +When UTF-8 is enabled, a positive value greater than 255 may be returned in +chptr. On entry, ptr is pointing at the \. On exit, it is on the final +character of the escape sequence. Arguments: ptrptr points to the pattern position pointer + chptr points to a returned data character errorcodeptr points to the errorcode variable bracount number of previous extracting brackets options the options bits isclass TRUE if inside a character class -Returns: zero or positive => a data character - negative => a special escape sequence +Returns: zero => a data character + positive => a special escape sequence + negative => a back reference on error, errorcodeptr is set */ static int -check_escape(const pcre_uchar **ptrptr, int *errorcodeptr, int bracount, - int options, BOOL isclass) +check_escape(const pcre_uchar **ptrptr, pcre_uint32 *chptr, int *errorcodeptr, + int bracount, int options, BOOL isclass) { /* PCRE_UTF16 has the same value as PCRE_UTF8. */ BOOL utf = (options & PCRE_UTF8) != 0; const pcre_uchar *ptr = *ptrptr + 1; -pcre_int32 c; +pcre_uint32 c; +int escape = 0; int i; GETCHARINCTEST(c, ptr); /* Get character value, increment pointer */ @@ -776,7 +1004,7 @@ ptr--; /* Set pointer back /* If backslash is at the end of the pattern, it's an error. */ -if (c == 0) *errorcodeptr = ERR1; +if (c == CHAR_NULL) *errorcodeptr = ERR1; /* Non-alphanumerics are literals. For digits or letters, do an initial lookup in a table. A non-zero result is something that can be returned immediately. @@ -785,12 +1013,13 @@ Otherwise further processing may be required. */ #ifndef EBCDIC /* ASCII/UTF-8 coding */ /* Not alphanumeric */ else if (c < CHAR_0 || c > CHAR_z) {} -else if ((i = escapes[c - CHAR_0]) != 0) c = i; +else if ((i = escapes[c - CHAR_0]) != 0) + { if (i > 0) c = (pcre_uint32)i; else escape = -i; } #else /* EBCDIC coding */ /* Not alphanumeric */ -else if (c < 'a' || (!MAX_255(c) || (ebcdic_chartab[c] & 0x0E) == 0)) {} -else if ((i = escapes[c - 0x48]) != 0) c = i; +else if (c < CHAR_a || (!MAX_255(c) || (ebcdic_chartab[c] & 0x0E) == 0)) {} +else if ((i = escapes[c - 0x48]) != 0) { if (i > 0) c = (pcre_uint32)i; else escape = -i; } #endif /* Escapes that need further processing, or are illegal. */ @@ -798,7 +1027,8 @@ else if ((i = escapes[c - 0x48]) != 0) c = i; else { const pcre_uchar *oldptr; - BOOL braced, negated; + BOOL braced, negated, overflow; + int s; switch (c) { @@ -823,7 +1053,7 @@ else c = 0; for (i = 0; i < 4; ++i) { - register int cc = *(++ptr); + register pcre_uint32 cc = *(++ptr); #ifndef EBCDIC /* ASCII/UTF-8 coding */ if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); @@ -833,13 +1063,13 @@ else #endif } -#ifdef COMPILE_PCRE8 - if (c > (utf ? 0x10ffff : 0xff)) -#else -#ifdef COMPILE_PCRE16 - if (c > (utf ? 0x10ffff : 0xffff)) +#if defined COMPILE_PCRE8 + if (c > (utf ? 0x10ffffU : 0xffU)) +#elif defined COMPILE_PCRE16 + if (c > (utf ? 0x10ffffU : 0xffffU)) +#elif defined COMPILE_PCRE32 + if (utf && c > 0x10ffffU) #endif -#endif { *errorcodeptr = ERR76; } @@ -870,13 +1100,13 @@ else (3) For Oniguruma compatibility we also support \g followed by a name or a number either in angle brackets or in single quotes. However, these are (possibly recursive) subroutine calls, _not_ backreferences. Just return - the -ESC_g code (cf \k). */ + the ESC_g code (cf \k). */ case CHAR_g: if (isclass) break; if (ptr[1] == CHAR_LESS_THAN_SIGN || ptr[1] == CHAR_APOSTROPHE) { - c = -ESC_g; + escape = ESC_g; break; } @@ -885,11 +1115,11 @@ else if (ptr[1] == CHAR_LEFT_CURLY_BRACKET) { const pcre_uchar *p; - for (p = ptr+2; *p != 0 && *p != CHAR_RIGHT_CURLY_BRACKET; p++) + for (p = ptr+2; *p != CHAR_NULL && *p != CHAR_RIGHT_CURLY_BRACKET; p++) if (*p != CHAR_MINUS && !IS_DIGIT(*p)) break; - if (*p != 0 && *p != CHAR_RIGHT_CURLY_BRACKET) + if (*p != CHAR_NULL && *p != CHAR_RIGHT_CURLY_BRACKET) { - c = -ESC_k; + escape = ESC_k; break; } braced = TRUE; @@ -905,17 +1135,18 @@ else else negated = FALSE; /* The integer range is limited by the machine's int representation. */ - c = 0; + s = 0; + overflow = FALSE; while (IS_DIGIT(ptr[1])) { - if (((unsigned int)c) > INT_MAX / 10) /* Integer overflow */ + if (s > INT_MAX / 10 - 1) /* Integer overflow */ { - c = -1; + overflow = TRUE; break; } - c = c * 10 + *(++ptr) - CHAR_0; + s = s * 10 + (int)(*(++ptr) - CHAR_0); } - if (((unsigned int)c) > INT_MAX) /* Integer overflow */ + if (overflow) /* Integer overflow */ { while (IS_DIGIT(ptr[1])) ptr++; @@ -929,7 +1160,7 @@ else break; } - if (c == 0) + if (s == 0) { *errorcodeptr = ERR58; break; @@ -937,29 +1168,33 @@ else if (negated) { - if (c > bracount) + if (s > bracount) { *errorcodeptr = ERR15; break; } - c = bracount - (c - 1); + s = bracount - (s - 1); } - c = -(ESC_REF + c); + escape = -s; break; /* The handling of escape sequences consisting of a string of digits - starting with one that is not zero is not straightforward. By experiment, - the way Perl works seems to be as follows: + starting with one that is not zero is not straightforward. Perl has changed + over the years. Nowadays \g{} for backreferences and \o{} for octal are + recommended to avoid the ambiguities in the old syntax. Outside a character class, the digits are read as a decimal number. If the - number is less than 10, or if there are that many previous extracting - left brackets, then it is a back reference. Otherwise, up to three octal - digits are read to form an escaped byte. Thus \123 is likely to be octal - 123 (cf \0123, which is octal 012 followed by the literal 3). If the octal - value is greater than 377, the least significant 8 bits are taken. Inside a - character class, \ followed by a digit is always an octal number. */ + number is less than 8 (used to be 10), or if there are that many previous + extracting left brackets, then it is a back reference. Otherwise, up to + three octal digits are read to form an escaped byte. Thus \123 is likely to + be octal 123 (cf \0123, which is octal 012 followed by the literal 3). If + the octal value is greater than 377, the least significant 8 bits are + taken. \8 and \9 are treated as the literal characters 8 and 9. + Inside a character class, \ followed by a digit is always either a literal + 8 or 9 or an octal number. */ + case CHAR_1: case CHAR_2: case CHAR_3: case CHAR_4: case CHAR_5: case CHAR_6: case CHAR_7: case CHAR_8: case CHAR_9: @@ -967,42 +1202,41 @@ else { oldptr = ptr; /* The integer range is limited by the machine's int representation. */ - c -= CHAR_0; + s = (int)(c -CHAR_0); + overflow = FALSE; while (IS_DIGIT(ptr[1])) { - if (((unsigned int)c) > INT_MAX / 10) /* Integer overflow */ + if (s > INT_MAX / 10 - 1) /* Integer overflow */ { - c = -1; + overflow = TRUE; break; } - c = c * 10 + *(++ptr) - CHAR_0; + s = s * 10 + (int)(*(++ptr) - CHAR_0); } - if (((unsigned int)c) > INT_MAX) /* Integer overflow */ + if (overflow) /* Integer overflow */ { while (IS_DIGIT(ptr[1])) ptr++; *errorcodeptr = ERR61; break; } - if (c < 10 || c <= bracount) + if (s < 8 || s <= bracount) /* Check for back reference */ { - c = -(ESC_REF + c); + escape = -s; break; } ptr = oldptr; /* Put the pointer back and fall through */ } - /* Handle an octal number following \. If the first digit is 8 or 9, Perl - generates a binary zero byte and treats the digit as a following literal. - Thus we have to pull back the pointer by one. */ + /* Handle a digit following \ when the number is not a back reference. If + the first digit is 8 or 9, Perl used to generate a binary zero byte and + then treat the digit as a following literal. At least by Perl 5.18 this + changed so as not to insert the binary zero. */ - if ((c = *ptr) >= CHAR_8) - { - ptr--; - c = 0; - break; - } + if ((c = *ptr) >= CHAR_8) break; + /* Fall through with a digit less than 8 */ + /* \0 always starts an octal number, but we may drop through to here with a larger first octal digit. The original code used just to take the least significant 8 bits of octal numbers (I think this is what early Perls used @@ -1018,22 +1252,57 @@ else #endif break; - /* \x is complicated. \x{ddd} is a character number which can be greater - than 0xff in utf or non-8bit mode, but only if the ddd are hex digits. - If not, { is treated as a data character. */ + /* \o is a relatively new Perl feature, supporting a more general way of + specifying character codes in octal. The only supported form is \o{ddd}. */ + case CHAR_o: + if (ptr[1] != CHAR_LEFT_CURLY_BRACKET) *errorcodeptr = ERR81; else + { + ptr += 2; + c = 0; + overflow = FALSE; + while (*ptr >= CHAR_0 && *ptr <= CHAR_7) + { + register pcre_uint32 cc = *ptr++; + if (c == 0 && cc == CHAR_0) continue; /* Leading zeroes */ +#ifdef COMPILE_PCRE32 + if (c >= 0x20000000l) { overflow = TRUE; break; } +#endif + c = (c << 3) + cc - CHAR_0 ; +#if defined COMPILE_PCRE8 + if (c > (utf ? 0x10ffffU : 0xffU)) { overflow = TRUE; break; } +#elif defined COMPILE_PCRE16 + if (c > (utf ? 0x10ffffU : 0xffffU)) { overflow = TRUE; break; } +#elif defined COMPILE_PCRE32 + if (utf && c > 0x10ffffU) { overflow = TRUE; break; } +#endif + } + if (overflow) + { + while (*ptr >= CHAR_0 && *ptr <= CHAR_7) ptr++; + *errorcodeptr = ERR34; + } + else if (*ptr == CHAR_RIGHT_CURLY_BRACKET) + { + if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73; + } + else *errorcodeptr = ERR80; + } + break; + + /* \x is complicated. In JavaScript, \x must be followed by two hexadecimal + numbers. Otherwise it is a lowercase x letter. */ + case CHAR_x: if ((options & PCRE_JAVASCRIPT_COMPAT) != 0) { - /* In JavaScript, \x must be followed by two hexadecimal numbers. - Otherwise it is a lowercase x letter. */ if (MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0 && MAX_255(ptr[2]) && (digitab[ptr[2]] & ctype_xdigit) != 0) { c = 0; for (i = 0; i < 2; ++i) { - register int cc = *(++ptr); + register pcre_uint32 cc = *(++ptr); #ifndef EBCDIC /* ASCII/UTF-8 coding */ if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); @@ -1043,68 +1312,86 @@ else #endif } } - break; - } + } /* End JavaScript handling */ - if (ptr[1] == CHAR_LEFT_CURLY_BRACKET) - { - const pcre_uchar *pt = ptr + 2; + /* Handle \x in Perl's style. \x{ddd} is a character number which can be + greater than 0xff in utf or non-8bit mode, but only if the ddd are hex + digits. If not, { used to be treated as a data character. However, Perl + seems to read hex digits up to the first non-such, and ignore the rest, so + that, for example \x{zz} matches a binary zero. This seems crazy, so PCRE + now gives an error. */ - c = 0; - while (MAX_255(*pt) && (digitab[*pt] & ctype_xdigit) != 0) + else + { + if (ptr[1] == CHAR_LEFT_CURLY_BRACKET) { - register int cc = *pt++; - if (c == 0 && cc == CHAR_0) continue; /* Leading zeroes */ + ptr += 2; + c = 0; + overflow = FALSE; + while (MAX_255(*ptr) && (digitab[*ptr] & ctype_xdigit) != 0) + { + register pcre_uint32 cc = *ptr++; + if (c == 0 && cc == CHAR_0) continue; /* Leading zeroes */ +#ifdef COMPILE_PCRE32 + if (c >= 0x10000000l) { overflow = TRUE; break; } +#endif + #ifndef EBCDIC /* ASCII/UTF-8 coding */ - if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ - c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); + if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ + c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); #else /* EBCDIC coding */ - if (cc >= CHAR_a && cc <= CHAR_z) cc += 64; /* Convert to upper case */ - c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); + if (cc >= CHAR_a && cc <= CHAR_z) cc += 64; /* Convert to upper case */ + c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); #endif -#ifdef COMPILE_PCRE8 - if (c > (utf ? 0x10ffff : 0xff)) { c = -1; break; } -#else -#ifdef COMPILE_PCRE16 - if (c > (utf ? 0x10ffff : 0xffff)) { c = -1; break; } +#if defined COMPILE_PCRE8 + if (c > (utf ? 0x10ffffU : 0xffU)) { overflow = TRUE; break; } +#elif defined COMPILE_PCRE16 + if (c > (utf ? 0x10ffffU : 0xffffU)) { overflow = TRUE; break; } +#elif defined COMPILE_PCRE32 + if (utf && c > 0x10ffffU) { overflow = TRUE; break; } #endif -#endif - } + } - if (c < 0) - { - while (MAX_255(*pt) && (digitab[*pt] & ctype_xdigit) != 0) pt++; - *errorcodeptr = ERR34; - } + if (overflow) + { + while (MAX_255(*ptr) && (digitab[*ptr] & ctype_xdigit) != 0) ptr++; + *errorcodeptr = ERR34; + } - if (*pt == CHAR_RIGHT_CURLY_BRACKET) - { - if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73; - ptr = pt; - break; - } + else if (*ptr == CHAR_RIGHT_CURLY_BRACKET) + { + if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73; + } - /* If the sequence of hex digits does not end with '}', then we don't - recognize this construct; fall through to the normal \x handling. */ - } + /* If the sequence of hex digits does not end with '}', give an error. + We used just to recognize this construct and fall through to the normal + \x handling, but nowadays Perl gives an error, which seems much more + sensible, so we do too. */ - /* Read just a single-byte hex-defined char */ + else *errorcodeptr = ERR79; + } /* End of \x{} processing */ - c = 0; - while (i++ < 2 && MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0) - { - int cc; /* Some compilers don't like */ - cc = *(++ptr); /* ++ in initializers */ + /* Read a single-byte hex-defined char (up to two hex digits after \x) */ + + else + { + c = 0; + while (i++ < 2 && MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0) + { + pcre_uint32 cc; /* Some compilers don't like */ + cc = *(++ptr); /* ++ in initializers */ #ifndef EBCDIC /* ASCII/UTF-8 coding */ - if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ - c = c * 16 + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); + if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ + c = c * 16 + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); #else /* EBCDIC coding */ - if (cc <= CHAR_z) cc += 64; /* Convert to upper case */ - c = c * 16 + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); + if (cc <= CHAR_z) cc += 64; /* Convert to upper case */ + c = c * 16 + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); #endif - } + } + } /* End of \xdd handling */ + } /* End of Perl-style \x handling */ break; /* For \c, a following letter is upper-cased; then the 0x40 bit is flipped. @@ -1114,7 +1401,7 @@ else case CHAR_c: c = *(++ptr); - if (c == 0) + if (c == CHAR_NULL) { *errorcodeptr = ERR2; break; @@ -1154,19 +1441,20 @@ else newline". PCRE does not support \N{name}. However, it does support quantification such as \N{2,3}. */ -if (c == -ESC_N && ptr[1] == CHAR_LEFT_CURLY_BRACKET && +if (escape == ESC_N && ptr[1] == CHAR_LEFT_CURLY_BRACKET && !is_counted_repeat(ptr+2)) *errorcodeptr = ERR37; /* If PCRE_UCP is set, we change the values for \d etc. */ -if ((options & PCRE_UCP) != 0 && c <= -ESC_D && c >= -ESC_w) - c -= (ESC_DU - ESC_D); +if ((options & PCRE_UCP) != 0 && escape >= ESC_D && escape <= ESC_w) + escape += (ESC_DU - ESC_D); /* Set the pointer to the final character before returning. */ *ptrptr = ptr; -return c; +*chptr = c; +return escape; } @@ -1184,21 +1472,24 @@ escape sequence. Argument: ptrptr points to the pattern position pointer negptr points to a boolean that is set TRUE for negation else FALSE - dptr points to an int that is set to the detailed property value + ptypeptr points to an unsigned int that is set to the type value + pdataptr points to an unsigned int that is set to the detailed property value errorcodeptr points to the error code variable -Returns: type value from ucp_type_table, or -1 for an invalid type +Returns: TRUE if the type value was found, or FALSE for an invalid type */ -static int -get_ucp(const pcre_uchar **ptrptr, BOOL *negptr, int *dptr, int *errorcodeptr) +static BOOL +get_ucp(const pcre_uchar **ptrptr, BOOL *negptr, unsigned int *ptypeptr, + unsigned int *pdataptr, int *errorcodeptr) { -int c, i, bot, top; +pcre_uchar c; +int i, bot, top; const pcre_uchar *ptr = *ptrptr; pcre_uchar name[32]; c = *(++ptr); -if (c == 0) goto ERROR_RETURN; +if (c == CHAR_NULL) goto ERROR_RETURN; *negptr = FALSE; @@ -1215,7 +1506,7 @@ if (c == CHAR_LEFT_CURLY_BRACKET) for (i = 0; i < (int)(sizeof(name) / sizeof(pcre_uchar)) - 1; i++) { c = *(++ptr); - if (c == 0) goto ERROR_RETURN; + if (c == CHAR_NULL) goto ERROR_RETURN; if (c == CHAR_RIGHT_CURLY_BRACKET) break; name[i] = c; } @@ -1240,30 +1531,31 @@ top = PRIV(utt_size); while (bot < top) { + int r; i = (bot + top) >> 1; - c = STRCMP_UC_C8(name, PRIV(utt_names) + PRIV(utt)[i].name_offset); - if (c == 0) + r = STRCMP_UC_C8(name, PRIV(utt_names) + PRIV(utt)[i].name_offset); + if (r == 0) { - *dptr = PRIV(utt)[i].value; - return PRIV(utt)[i].type; + *ptypeptr = PRIV(utt)[i].type; + *pdataptr = PRIV(utt)[i].value; + return TRUE; } - if (c > 0) bot = i + 1; else top = i; + if (r > 0) bot = i + 1; else top = i; } *errorcodeptr = ERR47; *ptrptr = ptr; -return -1; +return FALSE; ERROR_RETURN: *errorcodeptr = ERR46; *ptrptr = ptr; -return -1; +return FALSE; } #endif - /************************************************* * Read repeat counts * *************************************************/ @@ -1292,7 +1584,7 @@ int max = -1; /* Read the minimum value and do a paranoid check: a negative value indicates an integer overflow. */ -while (IS_DIGIT(*p)) min = min * 10 + *p++ - CHAR_0; +while (IS_DIGIT(*p)) min = min * 10 + (int)(*p++ - CHAR_0); if (min < 0 || min > 65535) { *errorcodeptr = ERR5; @@ -1307,7 +1599,7 @@ if (*p == CHAR_RIGHT_CURLY_BRACKET) max = min; else if (*(++p) != CHAR_RIGHT_CURLY_BRACKET) { max = 0; - while(IS_DIGIT(*p)) max = max * 10 + *p++ - CHAR_0; + while(IS_DIGIT(*p)) max = max * 10 + (int)(*p++ - CHAR_0); if (max < 0 || max > 65535) { *errorcodeptr = ERR5; @@ -1332,301 +1624,6 @@ return p; /************************************************* -* Subroutine for finding forward reference * -*************************************************/ - -/* This recursive function is called only from find_parens() below. The -top-level call starts at the beginning of the pattern. All other calls must -start at a parenthesis. It scans along a pattern's text looking for capturing -subpatterns, and counting them. If it finds a named pattern that matches the -name it is given, it returns its number. Alternatively, if the name is NULL, it -returns when it reaches a given numbered subpattern. Recursion is used to keep -track of subpatterns that reset the capturing group numbers - the (?| feature. - -This function was originally called only from the second pass, in which we know -that if (?< or (?' or (?P< is encountered, the name will be correctly -terminated because that is checked in the first pass. There is now one call to -this function in the first pass, to check for a recursive back reference by -name (so that we can make the whole group atomic). In this case, we need check -only up to the current position in the pattern, and that is still OK because -and previous occurrences will have been checked. To make this work, the test -for "end of pattern" is a check against cd->end_pattern in the main loop, -instead of looking for a binary zero. This means that the special first-pass -call can adjust cd->end_pattern temporarily. (Checks for binary zero while -processing items within the loop are OK, because afterwards the main loop will -terminate.) - -Arguments: - ptrptr address of the current character pointer (updated) - cd compile background data - name name to seek, or NULL if seeking a numbered subpattern - lorn name length, or subpattern number if name is NULL - xmode TRUE if we are in /x mode - utf TRUE if we are in UTF-8 / UTF-16 mode - count pointer to the current capturing subpattern number (updated) - -Returns: the number of the named subpattern, or -1 if not found -*/ - -static int -find_parens_sub(pcre_uchar **ptrptr, compile_data *cd, const pcre_uchar *name, int lorn, - BOOL xmode, BOOL utf, int *count) -{ -pcre_uchar *ptr = *ptrptr; -int start_count = *count; -int hwm_count = start_count; -BOOL dup_parens = FALSE; - -/* If the first character is a parenthesis, check on the type of group we are -dealing with. The very first call may not start with a parenthesis. */ - -if (ptr[0] == CHAR_LEFT_PARENTHESIS) - { - /* Handle specials such as (*SKIP) or (*UTF8) etc. */ - - if (ptr[1] == CHAR_ASTERISK) ptr += 2; - - /* Handle a normal, unnamed capturing parenthesis. */ - - else if (ptr[1] != CHAR_QUESTION_MARK) - { - *count += 1; - if (name == NULL && *count == lorn) return *count; - ptr++; - } - - /* All cases now have (? at the start. Remember when we are in a group - where the parenthesis numbers are duplicated. */ - - else if (ptr[2] == CHAR_VERTICAL_LINE) - { - ptr += 3; - dup_parens = TRUE; - } - - /* Handle comments; all characters are allowed until a ket is reached. */ - - else if (ptr[2] == CHAR_NUMBER_SIGN) - { - for (ptr += 3; *ptr != 0; ptr++) if (*ptr == CHAR_RIGHT_PARENTHESIS) break; - goto FAIL_EXIT; - } - - /* Handle a condition. If it is an assertion, just carry on so that it - is processed as normal. If not, skip to the closing parenthesis of the - condition (there can't be any nested parens). */ - - else if (ptr[2] == CHAR_LEFT_PARENTHESIS) - { - ptr += 2; - if (ptr[1] != CHAR_QUESTION_MARK) - { - while (*ptr != 0 && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; - if (*ptr != 0) ptr++; - } - } - - /* Start with (? but not a condition. */ - - else - { - ptr += 2; - if (*ptr == CHAR_P) ptr++; /* Allow optional P */ - - /* We have to disambiguate (? for named groups */ - - if ((*ptr == CHAR_LESS_THAN_SIGN && ptr[1] != CHAR_EXCLAMATION_MARK && - ptr[1] != CHAR_EQUALS_SIGN) || *ptr == CHAR_APOSTROPHE) - { - int term; - const pcre_uchar *thisname; - *count += 1; - if (name == NULL && *count == lorn) return *count; - term = *ptr++; - if (term == CHAR_LESS_THAN_SIGN) term = CHAR_GREATER_THAN_SIGN; - thisname = ptr; - while (*ptr != term) ptr++; - if (name != NULL && lorn == ptr - thisname && - STRNCMP_UC_UC(name, thisname, lorn) == 0) - return *count; - term++; - } - } - } - -/* Past any initial parenthesis handling, scan for parentheses or vertical -bars. Stop if we get to cd->end_pattern. Note that this is important for the -first-pass call when this value is temporarily adjusted to stop at the current -position. So DO NOT change this to a test for binary zero. */ - -for (; ptr < cd->end_pattern; ptr++) - { - /* Skip over backslashed characters and also entire \Q...\E */ - - if (*ptr == CHAR_BACKSLASH) - { - if (*(++ptr) == 0) goto FAIL_EXIT; - if (*ptr == CHAR_Q) for (;;) - { - while (*(++ptr) != 0 && *ptr != CHAR_BACKSLASH) {}; - if (*ptr == 0) goto FAIL_EXIT; - if (*(++ptr) == CHAR_E) break; - } - continue; - } - - /* Skip over character classes; this logic must be similar to the way they - are handled for real. If the first character is '^', skip it. Also, if the - first few characters (either before or after ^) are \Q\E or \E we skip them - too. This makes for compatibility with Perl. Note the use of STR macros to - encode "Q\\E" so that it works in UTF-8 on EBCDIC platforms. */ - - if (*ptr == CHAR_LEFT_SQUARE_BRACKET) - { - BOOL negate_class = FALSE; - for (;;) - { - if (ptr[1] == CHAR_BACKSLASH) - { - if (ptr[2] == CHAR_E) - ptr+= 2; - else if (STRNCMP_UC_C8(ptr + 2, - STR_Q STR_BACKSLASH STR_E, 3) == 0) - ptr += 4; - else - break; - } - else if (!negate_class && ptr[1] == CHAR_CIRCUMFLEX_ACCENT) - { - negate_class = TRUE; - ptr++; - } - else break; - } - - /* If the next character is ']', it is a data character that must be - skipped, except in JavaScript compatibility mode. */ - - if (ptr[1] == CHAR_RIGHT_SQUARE_BRACKET && - (cd->external_options & PCRE_JAVASCRIPT_COMPAT) == 0) - ptr++; - - while (*(++ptr) != CHAR_RIGHT_SQUARE_BRACKET) - { - if (*ptr == 0) return -1; - if (*ptr == CHAR_BACKSLASH) - { - if (*(++ptr) == 0) goto FAIL_EXIT; - if (*ptr == CHAR_Q) for (;;) - { - while (*(++ptr) != 0 && *ptr != CHAR_BACKSLASH) {}; - if (*ptr == 0) goto FAIL_EXIT; - if (*(++ptr) == CHAR_E) break; - } - continue; - } - } - continue; - } - - /* Skip comments in /x mode */ - - if (xmode && *ptr == CHAR_NUMBER_SIGN) - { - ptr++; - while (*ptr != 0) - { - if (IS_NEWLINE(ptr)) { ptr += cd->nllen - 1; break; } - ptr++; -#ifdef SUPPORT_UTF - if (utf) FORWARDCHAR(ptr); -#endif - } - if (*ptr == 0) goto FAIL_EXIT; - continue; - } - - /* Check for the special metacharacters */ - - if (*ptr == CHAR_LEFT_PARENTHESIS) - { - int rc = find_parens_sub(&ptr, cd, name, lorn, xmode, utf, count); - if (rc > 0) return rc; - if (*ptr == 0) goto FAIL_EXIT; - } - - else if (*ptr == CHAR_RIGHT_PARENTHESIS) - { - if (dup_parens && *count < hwm_count) *count = hwm_count; - goto FAIL_EXIT; - } - - else if (*ptr == CHAR_VERTICAL_LINE && dup_parens) - { - if (*count > hwm_count) hwm_count = *count; - *count = start_count; - } - } - -FAIL_EXIT: -*ptrptr = ptr; -return -1; -} - - - - -/************************************************* -* Find forward referenced subpattern * -*************************************************/ - -/* This function scans along a pattern's text looking for capturing -subpatterns, and counting them. If it finds a named pattern that matches the -name it is given, it returns its number. Alternatively, if the name is NULL, it -returns when it reaches a given numbered subpattern. This is used for forward -references to subpatterns. We used to be able to start this scan from the -current compiling point, using the current count value from cd->bracount, and -do it all in a single loop, but the addition of the possibility of duplicate -subpattern numbers means that we have to scan from the very start, in order to -take account of such duplicates, and to use a recursive function to keep track -of the different types of group. - -Arguments: - cd compile background data - name name to seek, or NULL if seeking a numbered subpattern - lorn name length, or subpattern number if name is NULL - xmode TRUE if we are in /x mode - utf TRUE if we are in UTF-8 / UTF-16 mode - -Returns: the number of the found subpattern, or -1 if not found -*/ - -static int -find_parens(compile_data *cd, const pcre_uchar *name, int lorn, BOOL xmode, - BOOL utf) -{ -pcre_uchar *ptr = (pcre_uchar *)cd->start_pattern; -int count = 0; -int rc; - -/* If the pattern does not start with an opening parenthesis, the first call -to find_parens_sub() will scan right to the end (if necessary). However, if it -does start with a parenthesis, find_parens_sub() will return when it hits the -matching closing parens. That is why we have to have a loop. */ - -for (;;) - { - rc = find_parens_sub(&ptr, cd, name, lorn, xmode, utf, &count); - if (rc > 0 || *ptr++ == 0) break; - } - -return rc; -} - - - - -/************************************************* * Find first significant op code * *************************************************/ @@ -1665,9 +1662,9 @@ for (;;) case OP_CALLOUT: case OP_CREF: - case OP_NCREF: + case OP_DNCREF: case OP_RREF: - case OP_NRREF: + case OP_DNRREF: case OP_DEF: code += PRIV(OP_lengths)[*code]; break; @@ -1681,7 +1678,6 @@ for (;;) - /************************************************* * Find the fixed length of a branch * *************************************************/ @@ -1699,7 +1695,7 @@ and doing the check at the end; a flag specifies which Arguments: code points to the start of the pattern (the bracket) - utf TRUE in UTF-8 / UTF-16 mode + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode atend TRUE if called when the pattern is complete cd the "compile data" structure @@ -1725,7 +1721,7 @@ for (;;) { int d; pcre_uchar *ce, *cs; - register int op = *cc; + register pcre_uchar op = *cc; switch (op) { @@ -1805,13 +1801,13 @@ for (;;) case OP_COMMIT: case OP_CREF: case OP_DEF: + case OP_DNCREF: + case OP_DNRREF: case OP_DOLL: case OP_DOLLM: case OP_EOD: case OP_EODN: case OP_FAIL: - case OP_NCREF: - case OP_NRREF: case OP_NOT_WORD_BOUNDARY: case OP_PRUNE: case OP_REVERSE: @@ -1845,7 +1841,7 @@ for (;;) case OP_EXACTI: case OP_NOTEXACT: case OP_NOTEXACTI: - branchlength += GET2(cc,1); + branchlength += (int)GET2(cc,1); cc += 2 + IMM2_SIZE; #ifdef SUPPORT_UTF if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); @@ -1854,7 +1850,8 @@ for (;;) case OP_TYPEEXACT: branchlength += GET2(cc,1); - if (cc[1 + IMM2_SIZE] == OP_PROP || cc[1 + IMM2_SIZE] == OP_NOTPROP) cc += 2; + if (cc[1 + IMM2_SIZE] == OP_PROP || cc[1 + IMM2_SIZE] == OP_NOTPROP) + cc += 2; cc += 1 + IMM2_SIZE + 1; break; @@ -1889,30 +1886,38 @@ for (;;) /* Check a class for variable quantification */ -#if defined SUPPORT_UTF || defined COMPILE_PCRE16 - case OP_XCLASS: - cc += GET(cc, 1) - PRIV(OP_lengths)[OP_CLASS]; - /* Fall through */ -#endif - case OP_CLASS: case OP_NCLASS: +#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + case OP_XCLASS: + /* The original code caused an unsigned overflow in 64 bit systems, + so now we use a conditional statement. */ + if (op == OP_XCLASS) + cc += GET(cc, 1); + else + cc += PRIV(OP_lengths)[OP_CLASS]; +#else cc += PRIV(OP_lengths)[OP_CLASS]; +#endif switch (*cc) { - case OP_CRPLUS: - case OP_CRMINPLUS: case OP_CRSTAR: case OP_CRMINSTAR: + case OP_CRPLUS: + case OP_CRMINPLUS: case OP_CRQUERY: case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSPLUS: + case OP_CRPOSQUERY: return -1; case OP_CRRANGE: case OP_CRMINRANGE: + case OP_CRPOSRANGE: if (GET2(cc,1) != GET2(cc,1+IMM2_SIZE)) return -1; - branchlength += GET2(cc,1); + branchlength += (int)GET2(cc,1); cc += 1 + 2 * IMM2_SIZE; break; @@ -1979,6 +1984,8 @@ for (;;) case OP_QUERYI: case OP_REF: case OP_REFI: + case OP_DNREF: + case OP_DNREFI: case OP_SBRA: case OP_SBRAPOS: case OP_SCBRA: @@ -2015,7 +2022,6 @@ for (;;) - /************************************************* * Scan compiled regex for specific bracket * *************************************************/ @@ -2028,7 +2034,7 @@ length. Arguments: code points to start of expression - utf TRUE in UTF-8 / UTF-16 mode + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode number the required bracket number or negative to find a lookbehind Returns: pointer to the opcode for the bracket, or NULL if not found @@ -2039,7 +2045,7 @@ PRIV(find_bracket)(const pcre_uchar *code, BOOL utf, i { for (;;) { - register int c = *code; + register pcre_uchar c = *code; if (c == OP_END) return NULL; @@ -2062,7 +2068,7 @@ for (;;) else if (c == OP_CBRA || c == OP_SCBRA || c == OP_CBRAPOS || c == OP_SCBRAPOS) { - int n = GET2(code, 1+LINK_SIZE); + int n = (int)GET2(code, 1+LINK_SIZE); if (n == number) return (pcre_uchar *)code; code += PRIV(OP_lengths)[c]; } @@ -2092,16 +2098,13 @@ for (;;) case OP_TYPEMINUPTO: case OP_TYPEEXACT: case OP_TYPEPOSUPTO: - if (code[1 + IMM2_SIZE] == OP_PROP - || code[1 + IMM2_SIZE] == OP_NOTPROP) code += 2; + if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) + code += 2; break; case OP_MARK: case OP_PRUNE_ARG: case OP_SKIP_ARG: - code += code[1]; - break; - case OP_THEN_ARG: code += code[1]; break; @@ -2115,7 +2118,7 @@ for (;;) a multi-byte character. The length in the table is a minimum, so we have to arrange to skip the extra bytes. */ -#ifdef SUPPORT_UTF +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 if (utf) switch(c) { case OP_CHAR: @@ -2167,7 +2170,7 @@ instance of OP_RECURSE. Arguments: code points to start of expression - utf TRUE in UTF-8 / UTF-16 mode + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode Returns: pointer to the opcode for OP_RECURSE, or NULL if not found */ @@ -2177,7 +2180,7 @@ find_recurse(const pcre_uchar *code, BOOL utf) { for (;;) { - register int c = *code; + register pcre_uchar c = *code; if (c == OP_END) return NULL; if (c == OP_RECURSE) return code; @@ -2212,16 +2215,13 @@ for (;;) case OP_TYPEUPTO: case OP_TYPEMINUPTO: case OP_TYPEEXACT: - if (code[1 + IMM2_SIZE] == OP_PROP - || code[1 + IMM2_SIZE] == OP_NOTPROP) code += 2; + if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) + code += 2; break; case OP_MARK: case OP_PRUNE_ARG: case OP_SKIP_ARG: - code += code[1]; - break; - case OP_THEN_ARG: code += code[1]; break; @@ -2235,7 +2235,7 @@ for (;;) by a multi-byte character. The length in the table is a minimum, so we have to arrange to skip the extra bytes. */ -#ifdef SUPPORT_UTF +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 if (utf) switch(c) { case OP_CHAR: @@ -2321,17 +2321,25 @@ bracket whose current branch will already have been sc Arguments: code points to start of search endcode points to where to stop - utf TRUE if in UTF-8 / UTF-16 mode + utf TRUE if in UTF-8 / UTF-16 / UTF-32 mode cd contains pointers to tables etc. + recurses chain of recurse_check to catch mutual recursion Returns: TRUE if what is matched could be empty */ +typedef struct recurse_check { + struct recurse_check *prev; + const pcre_uchar *group; +} recurse_check; + static BOOL could_be_empty_branch(const pcre_uchar *code, const pcre_uchar *endcode, - BOOL utf, compile_data *cd) + BOOL utf, compile_data *cd, recurse_check *recurses) { -register int c; +register pcre_uchar c; +recurse_check this_recurse; + for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE); code < endcode; code = first_significant_code(code + PRIV(OP_lengths)[c], TRUE)) @@ -2359,25 +2367,50 @@ for (code = first_significant_code(code + PRIV(OP_leng if (c == OP_RECURSE) { - const pcre_uchar *scode; + const pcre_uchar *scode = cd->start_code + GET(code, 1); BOOL empty_branch; - /* Test for forward reference */ + /* Test for forward reference or uncompleted reference. This is disabled + when called to scan a completed pattern by setting cd->start_workspace to + NULL. */ - for (scode = cd->start_workspace; scode < cd->hwm; scode += LINK_SIZE) - if (GET(scode, 0) == code + 1 - cd->start_code) return TRUE; + if (cd->start_workspace != NULL) + { + const pcre_uchar *tcode; + for (tcode = cd->start_workspace; tcode < cd->hwm; tcode += LINK_SIZE) + if ((int)GET(tcode, 0) == (int)(code + 1 - cd->start_code)) return TRUE; + if (GET(scode, 1) == 0) return TRUE; /* Unclosed */ + } - /* Not a forward reference, test for completed backward reference */ + /* If we are scanning a completed pattern, there are no forward references + and all groups are complete. We need to detect whether this is a recursive + call, as otherwise there will be an infinite loop. If it is a recursion, + just skip over it. Simple recursions are easily detected. For mutual + recursions we keep a chain on the stack. */ - empty_branch = FALSE; - scode = cd->start_code + GET(code, 1); - if (GET(scode, 1) == 0) return TRUE; /* Unclosed */ + else + { + recurse_check *r = recurses; + const pcre_uchar *endgroup = scode; - /* Completed backwards reference */ + do endgroup += GET(endgroup, 1); while (*endgroup == OP_ALT); + if (code >= scode && code <= endgroup) continue; /* Simple recursion */ + for (r = recurses; r != NULL; r = r->prev) + if (r->group == scode) break; + if (r != NULL) continue; /* Mutual recursion */ + } + + /* Completed reference; scan the referenced group, remembering it on the + stack chain to detect mutual recursions. */ + + empty_branch = FALSE; + this_recurse.prev = recurses; + this_recurse.group = scode; + do { - if (could_be_empty_branch(scode, endcode, utf, cd)) + if (could_be_empty_branch(scode, endcode, utf, cd, &this_recurse)) { empty_branch = TRUE; break; @@ -2433,7 +2466,7 @@ for (code = first_significant_code(code + PRIV(OP_leng empty_branch = FALSE; do { - if (!empty_branch && could_be_empty_branch(code, endcode, utf, cd)) + if (!empty_branch && could_be_empty_branch(code, endcode, utf, cd, NULL)) empty_branch = TRUE; code += GET(code, 1); } @@ -2475,15 +2508,19 @@ for (code = first_significant_code(code + PRIV(OP_leng case OP_CRMINSTAR: case OP_CRQUERY: case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSQUERY: break; default: /* Non-repeat => class must match */ case OP_CRPLUS: /* These repeats aren't empty */ case OP_CRMINPLUS: + case OP_CRPOSPLUS: return FALSE; case OP_CRRANGE: case OP_CRMINRANGE: + case OP_CRPOSRANGE: if (GET2(ccode, 1) > 0) return FALSE; /* Minimum > 0 */ break; } @@ -2491,34 +2528,57 @@ for (code = first_significant_code(code + PRIV(OP_leng /* Opcodes that must match a character */ + case OP_ANY: + case OP_ALLANY: + case OP_ANYBYTE: + case OP_PROP: case OP_NOTPROP: + case OP_ANYNL: + + case OP_NOT_HSPACE: + case OP_HSPACE: + case OP_NOT_VSPACE: + case OP_VSPACE: case OP_EXTUNI: + case OP_NOT_DIGIT: case OP_DIGIT: case OP_NOT_WHITESPACE: case OP_WHITESPACE: case OP_NOT_WORDCHAR: case OP_WORDCHAR: - case OP_ANY: - case OP_ALLANY: - case OP_ANYBYTE: + case OP_CHAR: case OP_CHARI: case OP_NOT: case OP_NOTI: + case OP_PLUS: + case OP_PLUSI: case OP_MINPLUS: - case OP_POSPLUS: - case OP_EXACT: + case OP_MINPLUSI: + case OP_NOTPLUS: + case OP_NOTPLUSI: case OP_NOTMINPLUS: + case OP_NOTMINPLUSI: + + case OP_POSPLUS: + case OP_POSPLUSI: case OP_NOTPOSPLUS: + case OP_NOTPOSPLUSI: + + case OP_EXACT: + case OP_EXACTI: case OP_NOTEXACT: + case OP_NOTEXACTI: + case OP_TYPEPLUS: case OP_TYPEMINPLUS: case OP_TYPEPOSPLUS: case OP_TYPEEXACT: + return FALSE; /* These are going to continue, as they may be empty, but we have to @@ -2538,8 +2598,8 @@ for (code = first_significant_code(code + PRIV(OP_leng case OP_TYPEUPTO: case OP_TYPEMINUPTO: case OP_TYPEPOSUPTO: - if (code[1 + IMM2_SIZE] == OP_PROP - || code[1 + IMM2_SIZE] == OP_NOTPROP) code += 2; + if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) + code += 2; break; /* End of branch */ @@ -2552,30 +2612,58 @@ for (code = first_significant_code(code + PRIV(OP_leng return TRUE; /* In UTF-8 mode, STAR, MINSTAR, POSSTAR, QUERY, MINQUERY, POSQUERY, UPTO, - MINUPTO, and POSUPTO may be followed by a multibyte character */ + MINUPTO, and POSUPTO and their caseless and negative versions may be + followed by a multibyte character. */ -#ifdef SUPPORT_UTF +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 case OP_STAR: case OP_STARI: + case OP_NOTSTAR: + case OP_NOTSTARI: + case OP_MINSTAR: case OP_MINSTARI: + case OP_NOTMINSTAR: + case OP_NOTMINSTARI: + case OP_POSSTAR: case OP_POSSTARI: + case OP_NOTPOSSTAR: + case OP_NOTPOSSTARI: + case OP_QUERY: case OP_QUERYI: + case OP_NOTQUERY: + case OP_NOTQUERYI: + case OP_MINQUERY: case OP_MINQUERYI: + case OP_NOTMINQUERY: + case OP_NOTMINQUERYI: + case OP_POSQUERY: case OP_POSQUERYI: + case OP_NOTPOSQUERY: + case OP_NOTPOSQUERYI: + if (utf && HAS_EXTRALEN(code[1])) code += GET_EXTRALEN(code[1]); break; case OP_UPTO: case OP_UPTOI: + case OP_NOTUPTO: + case OP_NOTUPTOI: + case OP_MINUPTO: case OP_MINUPTOI: + case OP_NOTMINUPTO: + case OP_NOTMINUPTOI: + case OP_POSUPTO: case OP_POSUPTOI: + case OP_NOTPOSUPTO: + case OP_NOTPOSUPTOI: + if (utf && HAS_EXTRALEN(code[1 + IMM2_SIZE])) code += GET_EXTRALEN(code[1 + IMM2_SIZE]); break; #endif @@ -2586,9 +2674,6 @@ for (code = first_significant_code(code + PRIV(OP_leng case OP_MARK: case OP_PRUNE_ARG: case OP_SKIP_ARG: - code += code[1]; - break; - case OP_THEN_ARG: code += code[1]; break; @@ -2620,7 +2705,7 @@ Arguments: code points to start of the recursion endcode points to where to stop (current RECURSE item) bcptr points to the chain of current (unclosed) branch starts - utf TRUE if in UTF-8 / UTF-16 mode + utf TRUE if in UTF-8 / UTF-16 / UTF-32 mode cd pointers to tables etc Returns: TRUE if what is matched could be empty @@ -2632,7 +2717,7 @@ could_be_empty(const pcre_uchar *code, const pcre_ucha { while (bcptr != NULL && bcptr->current_branch >= code) { - if (!could_be_empty_branch(bcptr->current_branch, endcode, utf, cd)) + if (!could_be_empty_branch(bcptr->current_branch, endcode, utf, cd, NULL)) return FALSE; bcptr = bcptr->outer; } @@ -2642,6 +2727,1072 @@ return TRUE; /************************************************* +* Base opcode of repeated opcodes * +*************************************************/ + +/* Returns the base opcode for repeated single character type opcodes. If the +opcode is not a repeated character type, it returns with the original value. + +Arguments: c opcode +Returns: base opcode for the type +*/ + +static pcre_uchar +get_repeat_base(pcre_uchar c) +{ +return (c > OP_TYPEPOSUPTO)? c : + (c >= OP_TYPESTAR)? OP_TYPESTAR : + (c >= OP_NOTSTARI)? OP_NOTSTARI : + (c >= OP_NOTSTAR)? OP_NOTSTAR : + (c >= OP_STARI)? OP_STARI : + OP_STAR; +} + + + +#ifdef SUPPORT_UCP +/************************************************* +* Check a character and a property * +*************************************************/ + +/* This function is called by check_auto_possessive() when a property item +is adjacent to a fixed character. + +Arguments: + c the character + ptype the property type + pdata the data for the type + negated TRUE if it's a negated property (\P or \p{^) + +Returns: TRUE if auto-possessifying is OK +*/ + +static BOOL +check_char_prop(pcre_uint32 c, unsigned int ptype, unsigned int pdata, + BOOL negated) +{ +const pcre_uint32 *p; +const ucd_record *prop = GET_UCD(c); + +switch(ptype) + { + case PT_LAMP: + return (prop->chartype == ucp_Lu || + prop->chartype == ucp_Ll || + prop->chartype == ucp_Lt) == negated; + + case PT_GC: + return (pdata == PRIV(ucp_gentype)[prop->chartype]) == negated; + + case PT_PC: + return (pdata == prop->chartype) == negated; + + case PT_SC: + return (pdata == prop->script) == negated; + + /* These are specials */ + + case PT_ALNUM: + return (PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N) == negated; + + /* Perl space used to exclude VT, but from Perl 5.18 it is included, which + means that Perl space and POSIX space are now identical. PCRE was changed + at release 8.34. */ + + case PT_SPACE: /* Perl space */ + case PT_PXSPACE: /* POSIX space */ + switch(c) + { + HSPACE_CASES: + VSPACE_CASES: + return negated; + + default: + return (PRIV(ucp_gentype)[prop->chartype] == ucp_Z) == negated; + } + break; /* Control never reaches here */ + + case PT_WORD: + return (PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N || + c == CHAR_UNDERSCORE) == negated; + + case PT_CLIST: + p = PRIV(ucd_caseless_sets) + prop->caseset; + for (;;) + { + if (c < *p) return !negated; + if (c == *p++) return negated; + } + break; /* Control never reaches here */ + } + +return FALSE; +} +#endif /* SUPPORT_UCP */ + + + +/************************************************* +* Fill the character property list * +*************************************************/ + +/* Checks whether the code points to an opcode that can take part in auto- +possessification, and if so, fills a list with its properties. + +Arguments: + code points to start of expression + utf TRUE if in UTF-8 / UTF-16 / UTF-32 mode + fcc points to case-flipping table + list points to output list + list[0] will be filled with the opcode + list[1] will be non-zero if this opcode + can match an empty character string + list[2..7] depends on the opcode + +Returns: points to the start of the next opcode if *code is accepted + NULL if *code is not accepted +*/ + +static const pcre_uchar * +get_chr_property_list(const pcre_uchar *code, BOOL utf, + const pcre_uint8 *fcc, pcre_uint32 *list) +{ +pcre_uchar c = *code; +pcre_uchar base; +const pcre_uchar *end; +pcre_uint32 chr; + +#ifdef SUPPORT_UCP +pcre_uint32 *clist_dest; +const pcre_uint32 *clist_src; +#else +utf = utf; /* Suppress "unused parameter" compiler warning */ +#endif + +list[0] = c; +list[1] = FALSE; +code++; + +if (c >= OP_STAR && c <= OP_TYPEPOSUPTO) + { + base = get_repeat_base(c); + c -= (base - OP_STAR); + + if (c == OP_UPTO || c == OP_MINUPTO || c == OP_EXACT || c == OP_POSUPTO) + code += IMM2_SIZE; + + list[1] = (c != OP_PLUS && c != OP_MINPLUS && c != OP_EXACT && c != OP_POSPLUS); + + switch(base) + { + case OP_STAR: + list[0] = OP_CHAR; + break; + + case OP_STARI: + list[0] = OP_CHARI; + break; + + case OP_NOTSTAR: + list[0] = OP_NOT; + break; + + case OP_NOTSTARI: + list[0] = OP_NOTI; + break; + + case OP_TYPESTAR: + list[0] = *code; + code++; + break; + } + c = list[0]; + } + +switch(c) + { + case OP_NOT_DIGIT: + case OP_DIGIT: + case OP_NOT_WHITESPACE: + case OP_WHITESPACE: + case OP_NOT_WORDCHAR: + case OP_WORDCHAR: + case OP_ANY: + case OP_ALLANY: + case OP_ANYNL: + case OP_NOT_HSPACE: + case OP_HSPACE: + case OP_NOT_VSPACE: + case OP_VSPACE: + case OP_EXTUNI: + case OP_EODN: + case OP_EOD: + case OP_DOLL: + case OP_DOLLM: + return code; + + case OP_CHAR: + case OP_NOT: + GETCHARINCTEST(chr, code); + list[2] = chr; + list[3] = NOTACHAR; + return code; + + case OP_CHARI: + case OP_NOTI: + list[0] = (c == OP_CHARI) ? OP_CHAR : OP_NOT; + GETCHARINCTEST(chr, code); + list[2] = chr; + +#ifdef SUPPORT_UCP + if (chr < 128 || (chr < 256 && !utf)) + list[3] = fcc[chr]; + else + list[3] = UCD_OTHERCASE(chr); +#elif defined SUPPORT_UTF || !defined COMPILE_PCRE8 + list[3] = (chr < 256) ? fcc[chr] : chr; +#else + list[3] = fcc[chr]; +#endif + + /* The othercase might be the same value. */ + + if (chr == list[3]) + list[3] = NOTACHAR; + else + list[4] = NOTACHAR; + return code; + +#ifdef SUPPORT_UCP + case OP_PROP: + case OP_NOTPROP: + if (code[0] != PT_CLIST) + { + list[2] = code[0]; + list[3] = code[1]; + return code + 2; + } + + /* Convert only if we have enough space. */ + + clist_src = PRIV(ucd_caseless_sets) + code[1]; + clist_dest = list + 2; + code += 2; + + do { + if (clist_dest >= list + 8) + { + /* Early return if there is not enough space. This should never + happen, since all clists are shorter than 5 character now. */ + list[2] = code[0]; + list[3] = code[1]; + return code; + } + *clist_dest++ = *clist_src; + } + while(*clist_src++ != NOTACHAR); + + /* All characters are stored. The terminating NOTACHAR + is copied form the clist itself. */ + + list[0] = (c == OP_PROP) ? OP_CHAR : OP_NOT; + return code; +#endif + + case OP_NCLASS: + case OP_CLASS: +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + if (c == OP_XCLASS) + end = code + GET(code, 0) - 1; + else +#endif + end = code + 32 / sizeof(pcre_uchar); + + switch(*end) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSQUERY: + list[1] = TRUE; + end++; + break; + + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRPOSPLUS: + end++; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + list[1] = (GET2(end, 1) == 0); + end += 1 + 2 * IMM2_SIZE; + break; + } + list[2] = end - code; + return end; + } +return NULL; /* Opcode not accepted */ +} + + + +/************************************************* +* Scan further character sets for match * +*************************************************/ + +/* Checks whether the base and the current opcode have a common character, in +which case the base cannot be possessified. + +Arguments: + code points to the byte code + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode + cd static compile data + base_list the data list of the base opcode + +Returns: TRUE if the auto-possessification is possible +*/ + +static BOOL +compare_opcodes(const pcre_uchar *code, BOOL utf, const compile_data *cd, + const pcre_uint32 *base_list, const pcre_uchar *base_end) +{ +pcre_uchar c; +pcre_uint32 list[8]; +const pcre_uint32 *chr_ptr; +const pcre_uint32 *ochr_ptr; +const pcre_uint32 *list_ptr; +const pcre_uchar *next_code; +const pcre_uint8 *class_bitset; +const pcre_uint32 *set1, *set2, *set_end; +pcre_uint32 chr; +BOOL accepted, invert_bits; + +/* Note: the base_list[1] contains whether the current opcode has greedy +(represented by a non-zero value) quantifier. This is a different from +other character type lists, which stores here that the character iterator +matches to an empty string (also represented by a non-zero value). */ + +for(;;) + { + /* All operations move the code pointer forward. + Therefore infinite recursions are not possible. */ + + c = *code; + + /* Skip over callouts */ + + if (c == OP_CALLOUT) + { + code += PRIV(OP_lengths)[c]; + continue; + } + + if (c == OP_ALT) + { + do code += GET(code, 1); while (*code == OP_ALT); + c = *code; + } + + switch(c) + { + case OP_END: + case OP_KETRPOS: + /* TRUE only in greedy case. The non-greedy case could be replaced by + an OP_EXACT, but it is probably not worth it. (And note that OP_EXACT + uses more memory, which we cannot get at this stage.) */ + + return base_list[1] != 0; + + case OP_KET: + /* If the bracket is capturing, and referenced by an OP_RECURSE, or + it is an atomic sub-pattern (assert, once, etc.) the non-greedy case + cannot be converted to a possessive form. */ + + if (base_list[1] == 0) return FALSE; + + switch(*(code - GET(code, 1))) + { + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + case OP_ONCE: + case OP_ONCE_NC: + /* Atomic sub-patterns and assertions can always auto-possessify their + last iterator. */ + return TRUE; + } + + code += PRIV(OP_lengths)[c]; + continue; + + case OP_ONCE: + case OP_ONCE_NC: + case OP_BRA: + case OP_CBRA: + next_code = code + GET(code, 1); + code += PRIV(OP_lengths)[c]; + + while (*next_code == OP_ALT) + { + if (!compare_opcodes(code, utf, cd, base_list, base_end)) return FALSE; + code = next_code + 1 + LINK_SIZE; + next_code += GET(next_code, 1); + } + continue; + + case OP_BRAZERO: + case OP_BRAMINZERO: + + next_code = code + 1; + if (*next_code != OP_BRA && *next_code != OP_CBRA + && *next_code != OP_ONCE && *next_code != OP_ONCE_NC) return FALSE; + + do next_code += GET(next_code, 1); while (*next_code == OP_ALT); + + /* The bracket content will be checked by the + OP_BRA/OP_CBRA case above. */ + next_code += 1 + LINK_SIZE; + if (!compare_opcodes(next_code, utf, cd, base_list, base_end)) + return FALSE; + + code += PRIV(OP_lengths)[c]; + continue; + } + + /* Check for a supported opcode, and load its properties. */ + + code = get_chr_property_list(code, utf, cd->fcc, list); + if (code == NULL) return FALSE; /* Unsupported */ + + /* If either opcode is a small character list, set pointers for comparing + characters from that list with another list, or with a property. */ + + if (base_list[0] == OP_CHAR) + { + chr_ptr = base_list + 2; + list_ptr = list; + } + else if (list[0] == OP_CHAR) + { + chr_ptr = list + 2; + list_ptr = base_list; + } + + /* Character bitsets can also be compared to certain opcodes. */ + + else if (base_list[0] == OP_CLASS || list[0] == OP_CLASS +#ifdef COMPILE_PCRE8 + /* In 8 bit, non-UTF mode, OP_CLASS and OP_NCLASS are the same. */ + || (!utf && (base_list[0] == OP_NCLASS || list[0] == OP_NCLASS)) +#endif + ) + { +#ifdef COMPILE_PCRE8 + if (base_list[0] == OP_CLASS || (!utf && base_list[0] == OP_NCLASS)) +#else + if (base_list[0] == OP_CLASS) +#endif + { + set1 = (pcre_uint32 *)(base_end - base_list[2]); + list_ptr = list; + } + else + { + set1 = (pcre_uint32 *)(code - list[2]); + list_ptr = base_list; + } + + invert_bits = FALSE; + switch(list_ptr[0]) + { + case OP_CLASS: + case OP_NCLASS: + set2 = (pcre_uint32 *) + ((list_ptr == list ? code : base_end) - list_ptr[2]); + break; + + /* OP_XCLASS cannot be supported here, because its bitset + is not necessarily complete. E.g: [a-\0x{200}] is stored + as a character range, and the appropriate bits are not set. */ + + case OP_NOT_DIGIT: + invert_bits = TRUE; + /* Fall through */ + case OP_DIGIT: + set2 = (pcre_uint32 *)(cd->cbits + cbit_digit); + break; + + case OP_NOT_WHITESPACE: + invert_bits = TRUE; + /* Fall through */ + case OP_WHITESPACE: + set2 = (pcre_uint32 *)(cd->cbits + cbit_space); + break; + + case OP_NOT_WORDCHAR: + invert_bits = TRUE; + /* Fall through */ + case OP_WORDCHAR: + set2 = (pcre_uint32 *)(cd->cbits + cbit_word); + break; + + default: + return FALSE; + } + + /* Compare 4 bytes to improve speed. */ + set_end = set1 + (32 / 4); + if (invert_bits) + { + do + { + if ((*set1++ & ~(*set2++)) != 0) return FALSE; + } + while (set1 < set_end); + } + else + { + do + { + if ((*set1++ & *set2++) != 0) return FALSE; + } + while (set1 < set_end); + } + + if (list[1] == 0) return TRUE; + /* Might be an empty repeat. */ + continue; + } + + /* Some property combinations also acceptable. Unicode property opcodes are + processed specially; the rest can be handled with a lookup table. */ + + else + { + pcre_uint32 leftop, rightop; + + leftop = base_list[0]; + rightop = list[0]; + +#ifdef SUPPORT_UCP + accepted = FALSE; /* Always set in non-unicode case. */ + if (leftop == OP_PROP || leftop == OP_NOTPROP) + { + if (rightop == OP_EOD) + accepted = TRUE; + else if (rightop == OP_PROP || rightop == OP_NOTPROP) + { + int n; + const pcre_uint8 *p; + BOOL same = leftop == rightop; + BOOL lisprop = leftop == OP_PROP; + BOOL risprop = rightop == OP_PROP; + BOOL bothprop = lisprop && risprop; + + /* There's a table that specifies how each combination is to be + processed: + 0 Always return FALSE (never auto-possessify) + 1 Character groups are distinct (possessify if both are OP_PROP) + 2 Check character categories in the same group (general or particular) + 3 Return TRUE if the two opcodes are not the same + ... see comments below + */ + + n = propposstab[base_list[2]][list[2]]; + switch(n) + { + case 0: break; + case 1: accepted = bothprop; break; + case 2: accepted = (base_list[3] == list[3]) != same; break; + case 3: accepted = !same; break; + + case 4: /* Left general category, right particular category */ + accepted = risprop && catposstab[base_list[3]][list[3]] == same; + break; + + case 5: /* Right general category, left particular category */ + accepted = lisprop && catposstab[list[3]][base_list[3]] == same; + break; + + /* This code is logically tricky. Think hard before fiddling with it. + The posspropstab table has four entries per row. Each row relates to + one of PCRE's special properties such as ALNUM or SPACE or WORD. + Only WORD actually needs all four entries, but using repeats for the + others means they can all use the same code below. + + The first two entries in each row are Unicode general categories, and + apply always, because all the characters they include are part of the + PCRE character set. The third and fourth entries are a general and a + particular category, respectively, that include one or more relevant + characters. One or the other is used, depending on whether the check + is for a general or a particular category. However, in both cases the + category contains more characters than the specials that are defined + for the property being tested against. Therefore, it cannot be used + in a NOTPROP case. + + Example: the row for WORD contains ucp_L, ucp_N, ucp_P, ucp_Po. + Underscore is covered by ucp_P or ucp_Po. */ + + case 6: /* Left alphanum vs right general category */ + case 7: /* Left space vs right general category */ + case 8: /* Left word vs right general category */ + p = posspropstab[n-6]; + accepted = risprop && lisprop == + (list[3] != p[0] && + list[3] != p[1] && + (list[3] != p[2] || !lisprop)); + break; + + case 9: /* Right alphanum vs left general category */ + case 10: /* Right space vs left general category */ + case 11: /* Right word vs left general category */ + p = posspropstab[n-9]; + accepted = lisprop && risprop == + (base_list[3] != p[0] && + base_list[3] != p[1] && + (base_list[3] != p[2] || !risprop)); + break; + + case 12: /* Left alphanum vs right particular category */ + case 13: /* Left space vs right particular category */ + case 14: /* Left word vs right particular category */ + p = posspropstab[n-12]; + accepted = risprop && lisprop == + (catposstab[p[0]][list[3]] && + catposstab[p[1]][list[3]] && + (list[3] != p[3] || !lisprop)); + break; + + case 15: /* Right alphanum vs left particular category */ + case 16: /* Right space vs left particular category */ + case 17: /* Right word vs left particular category */ + p = posspropstab[n-15]; + accepted = lisprop && risprop == + (catposstab[p[0]][base_list[3]] && + catposstab[p[1]][base_list[3]] && + (base_list[3] != p[3] || !risprop)); + break; + } + } + } + + else +#endif /* SUPPORT_UCP */ + + accepted = leftop >= FIRST_AUTOTAB_OP && leftop <= LAST_AUTOTAB_LEFT_OP && + rightop >= FIRST_AUTOTAB_OP && rightop <= LAST_AUTOTAB_RIGHT_OP && + autoposstab[leftop - FIRST_AUTOTAB_OP][rightop - FIRST_AUTOTAB_OP]; + + if (!accepted) + return FALSE; + + if (list[1] == 0) return TRUE; + /* Might be an empty repeat. */ + continue; + } + + /* Control reaches here only if one of the items is a small character list. + All characters are checked against the other side. */ + + do + { + chr = *chr_ptr; + + switch(list_ptr[0]) + { + case OP_CHAR: + ochr_ptr = list_ptr + 2; + do + { + if (chr == *ochr_ptr) return FALSE; + ochr_ptr++; + } + while(*ochr_ptr != NOTACHAR); + break; + + case OP_NOT: + ochr_ptr = list_ptr + 2; + do + { + if (chr == *ochr_ptr) + break; + ochr_ptr++; + } + while(*ochr_ptr != NOTACHAR); + if (*ochr_ptr == NOTACHAR) return FALSE; /* Not found */ + break; + + /* Note that OP_DIGIT etc. are generated only when PCRE_UCP is *not* + set. When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */ + + case OP_DIGIT: + if (chr < 256 && (cd->ctypes[chr] & ctype_digit) != 0) return FALSE; + break; + + case OP_NOT_DIGIT: + if (chr > 255 || (cd->ctypes[chr] & ctype_digit) == 0) return FALSE; + break; + + case OP_WHITESPACE: + if (chr < 256 && (cd->ctypes[chr] & ctype_space) != 0) return FALSE; + break; + + case OP_NOT_WHITESPACE: + if (chr > 255 || (cd->ctypes[chr] & ctype_space) == 0) return FALSE; + break; + + case OP_WORDCHAR: + if (chr < 255 && (cd->ctypes[chr] & ctype_word) != 0) return FALSE; + break; + + case OP_NOT_WORDCHAR: + if (chr > 255 || (cd->ctypes[chr] & ctype_word) == 0) return FALSE; + break; + + case OP_HSPACE: + switch(chr) + { + HSPACE_CASES: return FALSE; + default: break; + } + break; + + case OP_NOT_HSPACE: + switch(chr) + { + HSPACE_CASES: break; + default: return FALSE; + } + break; + + case OP_ANYNL: + case OP_VSPACE: + switch(chr) + { + VSPACE_CASES: return FALSE; + default: break; + } + break; + + case OP_NOT_VSPACE: + switch(chr) + { + VSPACE_CASES: break; + default: return FALSE; + } + break; + + case OP_DOLL: + case OP_EODN: + switch (chr) + { + case CHAR_CR: + case CHAR_LF: + case CHAR_VT: + case CHAR_FF: + case CHAR_NEL: +#ifndef EBCDIC + case 0x2028: + case 0x2029: +#endif /* Not EBCDIC */ + return FALSE; + } + break; + + case OP_EOD: /* Can always possessify before \z */ + break; + +#ifdef SUPPORT_UCP + case OP_PROP: + case OP_NOTPROP: + if (!check_char_prop(chr, list_ptr[2], list_ptr[3], + list_ptr[0] == OP_NOTPROP)) + return FALSE; + break; +#endif + + case OP_NCLASS: + if (chr > 255) return FALSE; + /* Fall through */ + + case OP_CLASS: + if (chr > 255) break; + class_bitset = (pcre_uint8 *) + ((list_ptr == list ? code : base_end) - list_ptr[2]); + if ((class_bitset[chr >> 3] & (1 << (chr & 7))) != 0) return FALSE; + break; + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + if (PRIV(xclass)(chr, (list_ptr == list ? code : base_end) - + list_ptr[2] + LINK_SIZE, utf)) return FALSE; + break; +#endif + + default: + return FALSE; + } + + chr_ptr++; + } + while(*chr_ptr != NOTACHAR); + + /* At least one character must be matched from this opcode. */ + + if (list[1] == 0) return TRUE; + } + +return FALSE; +} + + + +/************************************************* +* Scan compiled regex for auto-possession * +*************************************************/ + +/* Replaces single character iterations with their possessive alternatives +if appropriate. This function modifies the compiled opcode! + +Arguments: + code points to start of the byte code + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode + cd static compile data + +Returns: nothing +*/ + +static void +auto_possessify(pcre_uchar *code, BOOL utf, const compile_data *cd) +{ +register pcre_uchar c; +const pcre_uchar *end; +pcre_uchar *repeat_opcode; +pcre_uint32 list[8]; + +for (;;) + { + c = *code; + + if (c >= OP_STAR && c <= OP_TYPEPOSUPTO) + { + c -= get_repeat_base(c) - OP_STAR; + end = (c <= OP_MINUPTO) ? + get_chr_property_list(code, utf, cd->fcc, list) : NULL; + list[1] = c == OP_STAR || c == OP_PLUS || c == OP_QUERY || c == OP_UPTO; + + if (end != NULL && compare_opcodes(end, utf, cd, list, end)) + { + switch(c) + { + case OP_STAR: + *code += OP_POSSTAR - OP_STAR; + break; + + case OP_MINSTAR: + *code += OP_POSSTAR - OP_MINSTAR; + break; + + case OP_PLUS: + *code += OP_POSPLUS - OP_PLUS; + break; + + case OP_MINPLUS: + *code += OP_POSPLUS - OP_MINPLUS; + break; + + case OP_QUERY: + *code += OP_POSQUERY - OP_QUERY; + break; + + case OP_MINQUERY: + *code += OP_POSQUERY - OP_MINQUERY; + break; + + case OP_UPTO: + *code += OP_POSUPTO - OP_UPTO; + break; + + case OP_MINUPTO: + *code += OP_MINUPTO - OP_UPTO; + break; + } + } + c = *code; + } + else if (c == OP_CLASS || c == OP_NCLASS || c == OP_XCLASS) + { +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + if (c == OP_XCLASS) + repeat_opcode = code + GET(code, 1); + else +#endif + repeat_opcode = code + 1 + (32 / sizeof(pcre_uchar)); + + c = *repeat_opcode; + if (c >= OP_CRSTAR && c <= OP_CRMINRANGE) + { + /* end must not be NULL. */ + end = get_chr_property_list(code, utf, cd->fcc, list); + + list[1] = (c & 1) == 0; + + if (compare_opcodes(end, utf, cd, list, end)) + { + switch (c) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + *repeat_opcode = OP_CRPOSSTAR; + break; + + case OP_CRPLUS: + case OP_CRMINPLUS: + *repeat_opcode = OP_CRPOSPLUS; + break; + + case OP_CRQUERY: + case OP_CRMINQUERY: + *repeat_opcode = OP_CRPOSQUERY; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + *repeat_opcode = OP_CRPOSRANGE; + break; + } + } + } + c = *code; + } + + switch(c) + { + case OP_END: + return; + + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSPLUS: + case OP_TYPEPOSQUERY: + if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2; + break; + + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEEXACT: + case OP_TYPEPOSUPTO: + if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) + code += 2; + break; + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + code += GET(code, 1); + break; +#endif + + case OP_MARK: + case OP_PRUNE_ARG: + case OP_SKIP_ARG: + case OP_THEN_ARG: + code += code[1]; + break; + } + + /* Add in the fixed length from the table */ + + code += PRIV(OP_lengths)[c]; + + /* In UTF-8 mode, opcodes that are followed by a character may be followed by + a multi-byte character. The length in the table is a minimum, so we have to + arrange to skip the extra bytes. */ + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf) switch(c) + { + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + case OP_STAR: + case OP_MINSTAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_QUERY: + case OP_MINQUERY: + case OP_UPTO: + case OP_MINUPTO: + case OP_EXACT: + case OP_POSSTAR: + case OP_POSPLUS: + case OP_POSQUERY: + case OP_POSUPTO: + case OP_STARI: + case OP_MINSTARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_QUERYI: + case OP_MINQUERYI: + case OP_UPTOI: + case OP_MINUPTOI: + case OP_EXACTI: + case OP_POSSTARI: + case OP_POSPLUSI: + case OP_POSQUERYI: + case OP_POSUPTOI: + case OP_NOTSTAR: + case OP_NOTMINSTAR: + case OP_NOTPLUS: + case OP_NOTMINPLUS: + case OP_NOTQUERY: + case OP_NOTMINQUERY: + case OP_NOTUPTO: + case OP_NOTMINUPTO: + case OP_NOTEXACT: + case OP_NOTPOSSTAR: + case OP_NOTPOSPLUS: + case OP_NOTPOSQUERY: + case OP_NOTPOSUPTO: + case OP_NOTSTARI: + case OP_NOTMINSTARI: + case OP_NOTPLUSI: + case OP_NOTMINPLUSI: + case OP_NOTQUERYI: + case OP_NOTMINQUERYI: + case OP_NOTUPTOI: + case OP_NOTMINUPTOI: + case OP_NOTEXACTI: + case OP_NOTPOSSTARI: + case OP_NOTPOSPLUSI: + case OP_NOTPOSQUERYI: + case OP_NOTPOSUPTOI: + if (HAS_EXTRALEN(code[-1])) code += GET_EXTRALEN(code[-1]); + break; + } +#else + (void)(utf); /* Keep compiler happy by referencing function argument */ +#endif + } +} + + + +/************************************************* * Check for POSIX class syntax * *************************************************/ @@ -2662,7 +3813,7 @@ class, but [abc[:x\]pqr:]] is (so that an error can be below handles the special case of \], but does not try to do any other escape processing. This makes it different from Perl for cases such as [:l\ower:] where Perl recognizes it as the POSIX class "lower" but PCRE does not recognize -"l\ower". This is a lesser evil that not diagnosing bad classes when Perl does, +"l\ower". This is a lesser evil than not diagnosing bad classes when Perl does, I think. A user pointed out that PCRE was rejecting [:a[:digit:]] whereas Perl was not. @@ -2686,9 +3837,9 @@ Returns: TRUE or FALSE static BOOL check_posix_syntax(const pcre_uchar *ptr, const pcre_uchar **endptr) { -int terminator; /* Don't combine these lines; the Solaris cc */ +pcre_uchar terminator; /* Don't combine these lines; the Solaris cc */ terminator = *(++ptr); /* compiler warns about "non-constant" initializer. */ -for (++ptr; *ptr != 0; ptr++) +for (++ptr; *ptr != CHAR_NULL; ptr++) { if (*ptr == CHAR_BACKSLASH && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET) ptr++; @@ -2735,7 +3886,7 @@ register int yield = 0; while (posix_name_lengths[yield] != 0) { if (len == posix_name_lengths[yield] && - STRNCMP_UC_C8(ptr, pn, len) == 0) return yield; + STRNCMP_UC_C8(ptr, pn, (unsigned int)len) == 0) return yield; pn += posix_name_lengths[yield] + 1; yield++; } @@ -2767,7 +3918,7 @@ value in the reference (which is a group number). Arguments: group points to the start of the group adjust the amount by which the group is to be moved - utf TRUE in UTF-8 / UTF-16 mode + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode cd contains pointers to tables etc. save_hwm the hwm forward reference pointer at the start of the group @@ -2790,7 +3941,7 @@ while ((ptr = (pcre_uchar *)find_recurse(ptr, utf)) != for (hc = save_hwm; hc < cd->hwm; hc += LINK_SIZE) { - offset = GET(hc, 0); + offset = (int)GET(hc, 0); if (cd->start_code + offset == ptr + 1) { PUT(hc, 0, offset + adjust); @@ -2803,7 +3954,7 @@ while ((ptr = (pcre_uchar *)find_recurse(ptr, utf)) != if (hc >= cd->hwm) { - offset = GET(ptr, 1); + offset = (int)GET(ptr, 1); if (cd->start_code + offset >= group) PUT(ptr, 1, offset + adjust); } @@ -2871,9 +4022,10 @@ PUT(previous_callout, 2 + LINK_SIZE, length); *************************************************/ /* This function is passed the start and end of a class range, in UTF-8 mode -with UCP support. It searches up the characters, looking for internal ranges of +with UCP support. It searches up the characters, looking for ranges of characters in the "other" case. Each call returns the next one, updating the -start address. +start address. A character with multiple other cases is returned on its own +with a special return value. Arguments: cptr points to starting character value; updated @@ -2881,19 +4033,34 @@ Arguments: ocptr where to put start of othercase range odptr where to put end of othercase range -Yield: TRUE when range returned; FALSE when no more +Yield: -1 when no more + 0 when a range is returned + >0 the CASESET offset for char with multiple other cases + in this case, ocptr contains the original */ -static BOOL -get_othercase_range(unsigned int *cptr, unsigned int d, unsigned int *ocptr, - unsigned int *odptr) +static int +get_othercase_range(pcre_uint32 *cptr, pcre_uint32 d, pcre_uint32 *ocptr, + pcre_uint32 *odptr) { -unsigned int c, othercase, next; +pcre_uint32 c, othercase, next; +unsigned int co; +/* Find the first character that has an other case. If it has multiple other +cases, return its case offset value. */ + for (c = *cptr; c <= d; c++) - { if ((othercase = UCD_OTHERCASE(c)) != c) break; } + { + if ((co = UCD_CASESET(c)) != 0) + { + *ocptr = c++; /* Character that has the set */ + *cptr = c; /* Rest of input range */ + return (int)co; + } + if ((othercase = UCD_OTHERCASE(c)) != c) break; + } -if (c > d) return FALSE; +if (c > d) return -1; /* Reached end of range */ *ocptr = othercase; next = othercase + 1; @@ -2904,498 +4071,248 @@ for (++c; c <= d; c++) next++; } -*odptr = next - 1; -*cptr = c; - -return TRUE; +*odptr = next - 1; /* End of othercase range */ +*cptr = c; /* Rest of input range */ +return 0; } +#endif /* SUPPORT_UCP */ /************************************************* -* Check a character and a property * +* Add a character or range to a class * *************************************************/ -/* This function is called by check_auto_possessive() when a property item -is adjacent to a fixed character. +/* This function packages up the logic of adding a character or range of +characters to a class. The character values in the arguments will be within the +valid values for the current mode (8-bit, 16-bit, UTF, etc). This function is +mutually recursive with the function immediately below. Arguments: - c the character - ptype the property type - pdata the data for the type - negated TRUE if it's a negated property (\P or \p{^) + classbits the bit map for characters < 256 + uchardptr points to the pointer for extra data + options the options word + cd contains pointers to tables etc. + start start of range character + end end of range character -Returns: TRUE if auto-possessifying is OK +Returns: the number of < 256 characters added + the pointer to extra data is updated */ -static BOOL -check_char_prop(int c, int ptype, int pdata, BOOL negated) +static int +add_to_class(pcre_uint8 *classbits, pcre_uchar **uchardptr, int options, + compile_data *cd, pcre_uint32 start, pcre_uint32 end) { -const ucd_record *prop = GET_UCD(c); -switch(ptype) - { - case PT_LAMP: - return (prop->chartype == ucp_Lu || - prop->chartype == ucp_Ll || - prop->chartype == ucp_Lt) == negated; +pcre_uint32 c; +int n8 = 0; - case PT_GC: - return (pdata == PRIV(ucp_gentype)[prop->chartype]) == negated; +/* If caseless matching is required, scan the range and process alternate +cases. In Unicode, there are 8-bit characters that have alternate cases that +are greater than 255 and vice-versa. Sometimes we can just extend the original +range. */ - case PT_PC: - return (pdata == prop->chartype) == negated; +if ((options & PCRE_CASELESS) != 0) + { +#ifdef SUPPORT_UCP + if ((options & PCRE_UTF8) != 0) + { + int rc; + pcre_uint32 oc, od; - case PT_SC: - return (pdata == prop->script) == negated; + options &= ~PCRE_CASELESS; /* Remove for recursive calls */ + c = start; - /* These are specials */ + while ((rc = get_othercase_range(&c, end, &oc, &od)) >= 0) + { + /* Handle a single character that has more than one other case. */ - case PT_ALNUM: - return (PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N) == negated; + if (rc > 0) n8 += add_list_to_class(classbits, uchardptr, options, cd, + PRIV(ucd_caseless_sets) + rc, oc); - case PT_SPACE: /* Perl space */ - return (PRIV(ucp_gentype)[prop->chartype] == ucp_Z || - c == CHAR_HT || c == CHAR_NL || c == CHAR_FF || c == CHAR_CR) - == negated; + /* Do nothing if the other case range is within the original range. */ - case PT_PXSPACE: /* POSIX space */ - return (PRIV(ucp_gentype)[prop->chartype] == ucp_Z || - c == CHAR_HT || c == CHAR_NL || c == CHAR_VT || - c == CHAR_FF || c == CHAR_CR) - == negated; + else if (oc >= start && od <= end) continue; - case PT_WORD: - return (PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N || - c == CHAR_UNDERSCORE) == negated; - } -return FALSE; -} + /* Extend the original range if there is overlap, noting that if oc < c, we + can't have od > end because a subrange is always shorter than the basic + range. Otherwise, use a recursive call to add the additional range. */ + + else if (oc < start && od >= start - 1) start = oc; /* Extend downwards */ + else if (od > end && oc <= end + 1) end = od; /* Extend upwards */ + else n8 += add_to_class(classbits, uchardptr, options, cd, oc, od); + } + } + else #endif /* SUPPORT_UCP */ + /* Not UTF-mode, or no UCP */ - -/************************************************* -* Check if auto-possessifying is possible * -*************************************************/ - -/* This function is called for unlimited repeats of certain items, to see -whether the next thing could possibly match the repeated item. If not, it makes -sense to automatically possessify the repeated item. - -Arguments: - previous pointer to the repeated opcode - utf TRUE in UTF-8 / UTF-16 mode - ptr next character in pattern - options options bits - cd contains pointers to tables etc. - -Returns: TRUE if possessifying is wanted -*/ - -static BOOL -check_auto_possessive(const pcre_uchar *previous, BOOL utf, - const pcre_uchar *ptr, int options, compile_data *cd) -{ -pcre_int32 c, next; -int op_code = *previous++; - -/* Skip whitespace and comments in extended mode */ - -if ((options & PCRE_EXTENDED) != 0) - { - for (;;) + for (c = start; c <= end && c < 256; c++) { - while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_space) != 0) ptr++; - if (*ptr == CHAR_NUMBER_SIGN) - { - ptr++; - while (*ptr != 0) - { - if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; } - ptr++; -#ifdef SUPPORT_UTF - if (utf) FORWARDCHAR(ptr); -#endif - } - } - else break; + SETBIT(classbits, cd->fcc[c]); + n8++; } } -/* If the next item is one that we can handle, get its value. A non-negative -value is a character, a negative value is an escape value. */ +/* Now handle the original range. Adjust the final value according to the bit +length - this means that the same lists of (e.g.) horizontal spaces can be used +in all cases. */ -if (*ptr == CHAR_BACKSLASH) - { - int temperrorcode = 0; - next = check_escape(&ptr, &temperrorcode, cd->bracount, options, FALSE); - if (temperrorcode != 0) return FALSE; - ptr++; /* Point after the escape sequence */ - } -else if (!MAX_255(*ptr) || (cd->ctypes[*ptr] & ctype_meta) == 0) - { +#if defined COMPILE_PCRE8 #ifdef SUPPORT_UTF - if (utf) { GETCHARINC(next, ptr); } else + if ((options & PCRE_UTF8) == 0) #endif - next = *ptr++; - } -else return FALSE; + if (end > 0xff) end = 0xff; -/* Skip whitespace and comments in extended mode */ - -if ((options & PCRE_EXTENDED) != 0) - { - for (;;) - { - while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_space) != 0) ptr++; - if (*ptr == CHAR_NUMBER_SIGN) - { - ptr++; - while (*ptr != 0) - { - if (IS_NEWLINE(ptr)) { ptr += cd->nllen; break; } - ptr++; +#elif defined COMPILE_PCRE16 #ifdef SUPPORT_UTF - if (utf) FORWARDCHAR(ptr); + if ((options & PCRE_UTF16) == 0) #endif - } - } - else break; - } - } + if (end > 0xffff) end = 0xffff; -/* If the next thing is itself optional, we have to give up. */ +#endif /* COMPILE_PCRE[8|16] */ -if (*ptr == CHAR_ASTERISK || *ptr == CHAR_QUESTION_MARK || - STRNCMP_UC_C8(ptr, STR_LEFT_CURLY_BRACKET STR_0 STR_COMMA, 3) == 0) - return FALSE; +/* If all characters are less than 256, use the bit map. Otherwise use extra +data. */ -/* Now compare the next item with the previous opcode. First, handle cases when -the next item is a character. */ - -if (next >= 0) switch(op_code) +if (end < 0x100) { - case OP_CHAR: -#ifdef SUPPORT_UTF - GETCHARTEST(c, previous); -#else - c = *previous; -#endif - return c != next; - - /* For CHARI (caseless character) we must check the other case. If we have - Unicode property support, we can use it to test the other case of - high-valued characters. */ - - case OP_CHARI: -#ifdef SUPPORT_UTF - GETCHARTEST(c, previous); -#else - c = *previous; -#endif - if (c == next) return FALSE; -#ifdef SUPPORT_UTF - if (utf) + for (c = start; c <= end; c++) { - unsigned int othercase; - if (next < 128) othercase = cd->fcc[next]; else -#ifdef SUPPORT_UCP - othercase = UCD_OTHERCASE((unsigned int)next); -#else - othercase = NOTACHAR; -#endif - return (unsigned int)c != othercase; + n8++; + SETBIT(classbits, c); } - else -#endif /* SUPPORT_UTF */ - return (c != TABLE_GET((unsigned int)next, cd->fcc, next)); /* Non-UTF-8 mode */ + } - case OP_NOT: -#ifdef SUPPORT_UTF - GETCHARTEST(c, previous); -#else - c = *previous; -#endif - return c == next; +else + { + pcre_uchar *uchardata = *uchardptr; - case OP_NOTI: #ifdef SUPPORT_UTF - GETCHARTEST(c, previous); -#else - c = *previous; -#endif - if (c == next) return TRUE; -#ifdef SUPPORT_UTF - if (utf) + if ((options & PCRE_UTF8) != 0) /* All UTFs use the same flag bit */ { - unsigned int othercase; - if (next < 128) othercase = cd->fcc[next]; else -#ifdef SUPPORT_UCP - othercase = UCD_OTHERCASE((unsigned int)next); -#else - othercase = NOTACHAR; -#endif - return (unsigned int)c == othercase; + if (start < end) + { + *uchardata++ = XCL_RANGE; + uchardata += PRIV(ord2utf)(start, uchardata); + uchardata += PRIV(ord2utf)(end, uchardata); + } + else if (start == end) + { + *uchardata++ = XCL_SINGLE; + uchardata += PRIV(ord2utf)(start, uchardata); + } } else #endif /* SUPPORT_UTF */ - return (c == TABLE_GET((unsigned int)next, cd->fcc, next)); /* Non-UTF-8 mode */ - /* Note that OP_DIGIT etc. are generated only when PCRE_UCP is *not* set. - When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */ + /* Without UTF support, character values are constrained by the bit length, + and can only be > 256 for 16-bit and 32-bit libraries. */ - case OP_DIGIT: - return next > 255 || (cd->ctypes[next] & ctype_digit) == 0; - - case OP_NOT_DIGIT: - return next <= 255 && (cd->ctypes[next] & ctype_digit) != 0; - - case OP_WHITESPACE: - return next > 255 || (cd->ctypes[next] & ctype_space) == 0; - - case OP_NOT_WHITESPACE: - return next <= 255 && (cd->ctypes[next] & ctype_space) != 0; - - case OP_WORDCHAR: - return next > 255 || (cd->ctypes[next] & ctype_word) == 0; - - case OP_NOT_WORDCHAR: - return next <= 255 && (cd->ctypes[next] & ctype_word) != 0; - - case OP_HSPACE: - case OP_NOT_HSPACE: - switch(next) +#ifdef COMPILE_PCRE8 + {} +#else + if (start < end) { - case 0x09: - case 0x20: - case 0xa0: - case 0x1680: - case 0x180e: - case 0x2000: - case 0x2001: - case 0x2002: - case 0x2003: - case 0x2004: - case 0x2005: - case 0x2006: - case 0x2007: - case 0x2008: - case 0x2009: - case 0x200A: - case 0x202f: - case 0x205f: - case 0x3000: - return op_code == OP_NOT_HSPACE; - default: - return op_code != OP_NOT_HSPACE; + *uchardata++ = XCL_RANGE; + *uchardata++ = start; + *uchardata++ = end; } - - case OP_ANYNL: - case OP_VSPACE: - case OP_NOT_VSPACE: - switch(next) + else if (start == end) { - case 0x0a: - case 0x0b: - case 0x0c: - case 0x0d: - case 0x85: - case 0x2028: - case 0x2029: - return op_code == OP_NOT_VSPACE; - default: - return op_code != OP_NOT_VSPACE; + *uchardata++ = XCL_SINGLE; + *uchardata++ = start; } - -#ifdef SUPPORT_UCP - case OP_PROP: - return check_char_prop(next, previous[0], previous[1], FALSE); - - case OP_NOTPROP: - return check_char_prop(next, previous[0], previous[1], TRUE); #endif - default: - return FALSE; + *uchardptr = uchardata; /* Updata extra data pointer */ } +return n8; /* Number of 8-bit characters */ +} -/* Handle the case when the next item is \d, \s, etc. Note that when PCRE_UCP -is set, \d turns into ESC_du rather than ESC_d, etc., so ESC_d etc. are -generated only when PCRE_UCP is *not* set, that is, when only ASCII -characteristics are recognized. Similarly, the opcodes OP_DIGIT etc. are -replaced by OP_PROP codes when PCRE_UCP is set. */ -switch(op_code) - { - case OP_CHAR: - case OP_CHARI: -#ifdef SUPPORT_UTF - GETCHARTEST(c, previous); -#else - c = *previous; -#endif - switch(-next) - { - case ESC_d: - return c > 255 || (cd->ctypes[c] & ctype_digit) == 0; - case ESC_D: - return c <= 255 && (cd->ctypes[c] & ctype_digit) != 0; - case ESC_s: - return c > 255 || (cd->ctypes[c] & ctype_space) == 0; +/************************************************* +* Add a list of characters to a class * +*************************************************/ - case ESC_S: - return c <= 255 && (cd->ctypes[c] & ctype_space) != 0; +/* This function is used for adding a list of case-equivalent characters to a +class, and also for adding a list of horizontal or vertical whitespace. If the +list is in order (which it should be), ranges of characters are detected and +handled appropriately. This function is mutually recursive with the function +above. - case ESC_w: - return c > 255 || (cd->ctypes[c] & ctype_word) == 0; +Arguments: + classbits the bit map for characters < 256 + uchardptr points to the pointer for extra data + options the options word + cd contains pointers to tables etc. + p points to row of 32-bit values, terminated by NOTACHAR + except character to omit; this is used when adding lists of + case-equivalent characters to avoid including the one we + already know about - case ESC_W: - return c <= 255 && (cd->ctypes[c] & ctype_word) != 0; +Returns: the number of < 256 characters added + the pointer to extra data is updated +*/ - case ESC_h: - case ESC_H: - switch(c) - { - case 0x09: - case 0x20: - case 0xa0: - case 0x1680: - case 0x180e: - case 0x2000: - case 0x2001: - case 0x2002: - case 0x2003: - case 0x2004: - case 0x2005: - case 0x2006: - case 0x2007: - case 0x2008: - case 0x2009: - case 0x200A: - case 0x202f: - case 0x205f: - case 0x3000: - return -next != ESC_h; - default: - return -next == ESC_h; - } - - case ESC_v: - case ESC_V: - switch(c) - { - case 0x0a: - case 0x0b: - case 0x0c: - case 0x0d: - case 0x85: - case 0x2028: - case 0x2029: - return -next != ESC_v; - default: - return -next == ESC_v; - } - - /* When PCRE_UCP is set, these values get generated for \d etc. Find - their substitutions and process them. The result will always be either - -ESC_p or -ESC_P. Then fall through to process those values. */ - -#ifdef SUPPORT_UCP - case ESC_du: - case ESC_DU: - case ESC_wu: - case ESC_WU: - case ESC_su: - case ESC_SU: - { - int temperrorcode = 0; - ptr = substitutes[-next - ESC_DU]; - next = check_escape(&ptr, &temperrorcode, 0, options, FALSE); - if (temperrorcode != 0) return FALSE; - ptr++; /* For compatibility */ - } - /* Fall through */ - - case ESC_p: - case ESC_P: - { - int ptype, pdata, errorcodeptr; - BOOL negated; - - ptr--; /* Make ptr point at the p or P */ - ptype = get_ucp(&ptr, &negated, &pdata, &errorcodeptr); - if (ptype < 0) return FALSE; - ptr++; /* Point past the final curly ket */ - - /* If the property item is optional, we have to give up. (When generated - from \d etc by PCRE_UCP, this test will have been applied much earlier, - to the original \d etc. At this point, ptr will point to a zero byte. */ - - if (*ptr == CHAR_ASTERISK || *ptr == CHAR_QUESTION_MARK || - STRNCMP_UC_C8(ptr, STR_LEFT_CURLY_BRACKET STR_0 STR_COMMA, 3) == 0) - return FALSE; - - /* Do the property check. */ - - return check_char_prop(c, ptype, pdata, (next == -ESC_P) != negated); - } -#endif - - default: - return FALSE; +static int +add_list_to_class(pcre_uint8 *classbits, pcre_uchar **uchardptr, int options, + compile_data *cd, const pcre_uint32 *p, unsigned int except) +{ +int n8 = 0; +while (p[0] < NOTACHAR) + { + int n = 0; + if (p[0] != except) + { + while(p[n+1] == p[0] + n + 1) n++; + n8 += add_to_class(classbits, uchardptr, options, cd, p[0], p[n]); } + p += n + 1; + } +return n8; +} - /* In principle, support for Unicode properties should be integrated here as - well. It means re-organizing the above code so as to get hold of the property - values before switching on the op-code. However, I wonder how many patterns - combine ASCII \d etc with Unicode properties? (Note that if PCRE_UCP is set, - these op-codes are never generated.) */ - case OP_DIGIT: - return next == -ESC_D || next == -ESC_s || next == -ESC_W || - next == -ESC_h || next == -ESC_v || next == -ESC_R; - case OP_NOT_DIGIT: - return next == -ESC_d; +/************************************************* +* Add characters not in a list to a class * +*************************************************/ - case OP_WHITESPACE: - return next == -ESC_S || next == -ESC_d || next == -ESC_w; +/* This function is used for adding the complement of a list of horizontal or +vertical whitespace to a class. The list must be in order. - case OP_NOT_WHITESPACE: - return next == -ESC_s || next == -ESC_h || next == -ESC_v || next == -ESC_R; +Arguments: + classbits the bit map for characters < 256 + uchardptr points to the pointer for extra data + options the options word + cd contains pointers to tables etc. + p points to row of 32-bit values, terminated by NOTACHAR - case OP_HSPACE: - return next == -ESC_S || next == -ESC_H || next == -ESC_d || - next == -ESC_w || next == -ESC_v || next == -ESC_R; +Returns: the number of < 256 characters added + the pointer to extra data is updated +*/ - case OP_NOT_HSPACE: - return next == -ESC_h; - - /* Can't have \S in here because VT matches \S (Perl anomaly) */ - case OP_ANYNL: - case OP_VSPACE: - return next == -ESC_V || next == -ESC_d || next == -ESC_w; - - case OP_NOT_VSPACE: - return next == -ESC_v || next == -ESC_R; - - case OP_WORDCHAR: - return next == -ESC_W || next == -ESC_s || next == -ESC_h || - next == -ESC_v || next == -ESC_R; - - case OP_NOT_WORDCHAR: - return next == -ESC_w || next == -ESC_d; - - default: - return FALSE; +static int +add_not_list_to_class(pcre_uint8 *classbits, pcre_uchar **uchardptr, + int options, compile_data *cd, const pcre_uint32 *p) +{ +BOOL utf = (options & PCRE_UTF8) != 0; +int n8 = 0; +if (p[0] > 0) + n8 += add_to_class(classbits, uchardptr, options, cd, 0, p[0] - 1); +while (p[0] < NOTACHAR) + { + while (p[1] == p[0] + 1) p++; + n8 += add_to_class(classbits, uchardptr, options, cd, p[0] + 1, + (p[1] == NOTACHAR) ? (utf ? 0x10ffffu : 0xffffffffu) : p[1] - 1); + p++; } - -/* Control does not reach here */ +return n8; } @@ -3411,39 +4328,46 @@ to find out the amount of memory needed, as well as du phase. The value of lengthptr distinguishes the two phases. Arguments: - optionsptr pointer to the option bits - codeptr points to the pointer to the current code point - ptrptr points to the current pattern pointer - errorcodeptr points to error code variable - firstcharptr set to initial literal character, or < 0 (REQ_UNSET, REQ_NONE) - reqcharptr set to the last literal character required, else < 0 - bcptr points to current branch chain - cond_depth conditional nesting depth - cd contains pointers to tables etc. - lengthptr NULL during the real compile phase - points to length accumulator during pre-compile phase + optionsptr pointer to the option bits + codeptr points to the pointer to the current code point + ptrptr points to the current pattern pointer + errorcodeptr points to error code variable + firstcharptr place to put the first required character + firstcharflagsptr place to put the first character flags, or a negative number + reqcharptr place to put the last required character + reqcharflagsptr place to put the last required character flags, or a negative number + bcptr points to current branch chain + cond_depth conditional nesting depth + cd contains pointers to tables etc. + lengthptr NULL during the real compile phase + points to length accumulator during pre-compile phase -Returns: TRUE on success - FALSE, with *errorcodeptr set non-zero on error +Returns: TRUE on success + FALSE, with *errorcodeptr set non-zero on error */ static BOOL compile_branch(int *optionsptr, pcre_uchar **codeptr, - const pcre_uchar **ptrptr, int *errorcodeptr, pcre_int32 *firstcharptr, - pcre_int32 *reqcharptr, branch_chain *bcptr, int cond_depth, + const pcre_uchar **ptrptr, int *errorcodeptr, + pcre_uint32 *firstcharptr, pcre_int32 *firstcharflagsptr, + pcre_uint32 *reqcharptr, pcre_int32 *reqcharflagsptr, + branch_chain *bcptr, int cond_depth, compile_data *cd, int *lengthptr) { int repeat_type, op_type; int repeat_min = 0, repeat_max = 0; /* To please picky compilers */ int bravalue = 0; int greedy_default, greedy_non_default; -pcre_int32 firstchar, reqchar; -pcre_int32 zeroreqchar, zerofirstchar; +pcre_uint32 firstchar, reqchar; +pcre_int32 firstcharflags, reqcharflags; +pcre_uint32 zeroreqchar, zerofirstchar; +pcre_int32 zeroreqcharflags, zerofirstcharflags; pcre_int32 req_caseopt, reqvary, tempreqvary; int options = *optionsptr; /* May change dynamically */ int after_manual_callout = 0; int length_prevgroup = 0; -register int c; +register pcre_uint32 c; +int escape; register pcre_uchar *code = *codeptr; pcre_uchar *last_code = code; pcre_uchar *orig_code = code; @@ -3463,18 +4387,23 @@ must not do this for other options (e.g. PCRE_EXTENDED dynamically as we process the pattern. */ #ifdef SUPPORT_UTF -/* PCRE_UTF16 has the same value as PCRE_UTF8. */ +/* PCRE_UTF[16|32] have the same value as PCRE_UTF8. */ BOOL utf = (options & PCRE_UTF8) != 0; +#ifndef COMPILE_PCRE32 pcre_uchar utf_chars[6]; +#endif #else BOOL utf = FALSE; #endif -/* Helper variables for OP_XCLASS opcode (for characters > 255). */ +/* Helper variables for OP_XCLASS opcode (for characters > 255). We define +class_uchardata always so that it can be passed to add_to_class() always, +though it will not be used in non-UTF 8-bit cases. This avoids having to supply +alternative calls for the different cases. */ +pcre_uchar *class_uchardata; #if defined SUPPORT_UTF || !defined COMPILE_PCRE8 BOOL xclass; -pcre_uchar *class_uchardata; pcre_uchar *class_uchardata_base; #endif @@ -3497,7 +4426,8 @@ to take the zero repeat into account. This is implemen zerofirstbyte and zeroreqchar when such a repeat is encountered. The individual item types that can be repeated set these backoff variables appropriately. */ -firstchar = reqchar = zerofirstchar = zeroreqchar = REQ_UNSET; +firstchar = reqchar = zerofirstchar = zeroreqchar = 0; +firstcharflags = reqcharflags = zerofirstcharflags = zeroreqcharflags = REQ_UNSET; /* The variable req_caseopt contains either the REQ_CASELESS value or zero, according to the current setting of the caseless flag. The @@ -3518,16 +4448,17 @@ for (;; ptr++) BOOL is_recurse; BOOL reset_bracount; int class_has_8bitchar; - int class_single_char; + int class_one_char; int newoptions; int recno; int refsign; int skipbytes; - int subreqchar; - int subfirstchar; + pcre_uint32 subreqchar, subfirstchar; + pcre_int32 subreqcharflags, subfirstcharflags; int terminator; - int mclength; - int tempbracount; + unsigned int mclength; + unsigned int tempbracount; + pcre_uint32 ec; pcre_uchar mcbuffer[8]; /* Get next character in the pattern */ @@ -3537,7 +4468,7 @@ for (;; ptr++) /* If we are at the end of a nested substitution, revert to the outer level string. Nesting only happens one level deep. */ - if (c == 0 && nestptr != NULL) + if (c == CHAR_NULL && nestptr != NULL) { ptr = nestptr; nestptr = NULL; @@ -3612,7 +4543,7 @@ for (;; ptr++) /* If in \Q...\E, check for the end; if not, we have a literal */ - if (inescq && c != 0) + if (inescq && c != CHAR_NULL) { if (c == CHAR_BACKSLASH && ptr[1] == CHAR_E) { @@ -3635,62 +4566,73 @@ for (;; ptr++) } goto NORMAL_CHAR; } + /* Control does not reach here. */ } - /* Fill in length of a previous callout, except when the next thing is - a quantifier. */ + /* In extended mode, skip white space and comments. We need a loop in order + to check for more white space and more comments after a comment. */ - is_quantifier = - c == CHAR_ASTERISK || c == CHAR_PLUS || c == CHAR_QUESTION_MARK || - (c == CHAR_LEFT_CURLY_BRACKET && is_counted_repeat(ptr+1)); - - if (!is_quantifier && previous_callout != NULL && - after_manual_callout-- <= 0) - { - if (lengthptr == NULL) /* Don't attempt in pre-compile phase */ - complete_callout(previous_callout, ptr, cd); - previous_callout = NULL; - } - - /* In extended mode, skip white space and comments. */ - if ((options & PCRE_EXTENDED) != 0) { - if (MAX_255(*ptr) && (cd->ctypes[c] & ctype_space) != 0) continue; - if (c == CHAR_NUMBER_SIGN) + for (;;) { + while (MAX_255(c) && (cd->ctypes[c] & ctype_space) != 0) c = *(++ptr); + if (c != CHAR_NUMBER_SIGN) break; ptr++; - while (*ptr != 0) + while (*ptr != CHAR_NULL) { - if (IS_NEWLINE(ptr)) { ptr += cd->nllen - 1; break; } + if (IS_NEWLINE(ptr)) /* For non-fixed-length newline cases, */ + { /* IS_NEWLINE sets cd->nllen. */ + ptr += cd->nllen; + break; + } ptr++; #ifdef SUPPORT_UTF if (utf) FORWARDCHAR(ptr); #endif } - if (*ptr != 0) continue; - - /* Else fall through to handle end of string */ - c = 0; + c = *ptr; /* Either NULL or the char after a newline */ } } - /* No auto callout for quantifiers. */ + /* See if the next thing is a quantifier. */ - if ((options & PCRE_AUTO_CALLOUT) != 0 && !is_quantifier) + is_quantifier = + c == CHAR_ASTERISK || c == CHAR_PLUS || c == CHAR_QUESTION_MARK || + (c == CHAR_LEFT_CURLY_BRACKET && is_counted_repeat(ptr+1)); + + /* Fill in length of a previous callout, except when the next thing is a + quantifier or when processing a property substitution string in UCP mode. */ + + if (!is_quantifier && previous_callout != NULL && nestptr == NULL && + after_manual_callout-- <= 0) { + if (lengthptr == NULL) /* Don't attempt in pre-compile phase */ + complete_callout(previous_callout, ptr, cd); + previous_callout = NULL; + } + + /* Create auto callout, except for quantifiers, or while processing property + strings that are substituted for \w etc in UCP mode. */ + + if ((options & PCRE_AUTO_CALLOUT) != 0 && !is_quantifier && nestptr == NULL) + { previous_callout = code; code = auto_callout(code, ptr, cd); } + /* Process the next pattern item. */ + switch(c) { /* ===================================================================*/ - case 0: /* The branch terminates at string end */ + case CHAR_NULL: /* The branch terminates at string end */ case CHAR_VERTICAL_LINE: /* or | or ) */ case CHAR_RIGHT_PARENTHESIS: *firstcharptr = firstchar; + *firstcharflagsptr = firstcharflags; *reqcharptr = reqchar; + *reqcharflagsptr = reqcharflags; *codeptr = code; *ptrptr = ptr; if (lengthptr != NULL) @@ -3714,7 +4656,7 @@ for (;; ptr++) previous = NULL; if ((options & PCRE_MULTILINE) != 0) { - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; *code++ = OP_CIRCM; } else *code++ = OP_CIRC; @@ -3729,9 +4671,11 @@ for (;; ptr++) repeats. The value of reqchar doesn't change either. */ case CHAR_DOT: - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; previous = code; *code++ = ((options & PCRE_DOTALL) != 0)? OP_ALLANY: OP_ANY; break; @@ -3760,7 +4704,29 @@ for (;; ptr++) } goto NORMAL_CHAR; + /* In another (POSIX) regex library, the ugly syntax [[:<:]] and [[:>:]] is + used for "start of word" and "end of word". As these are otherwise illegal + sequences, we don't break anything by recognizing them. They are replaced + by \b(?=\w) and \b(?<=\w) respectively. Sequences like [a[:<:]] are + erroneous and are handled by the normal code below. */ + case CHAR_LEFT_SQUARE_BRACKET: + if (STRNCMP_UC_C8(ptr+1, STRING_WEIRD_STARTWORD, 6) == 0) + { + nestptr = ptr + 7; + ptr = sub_start_of_word - 1; + continue; + } + + if (STRNCMP_UC_C8(ptr+1, STRING_WEIRD_ENDWORD, 6) == 0) + { + nestptr = ptr + 7; + ptr = sub_end_of_word - 1; + continue; + } + + /* Handle a real character class. */ + previous = code; /* PCRE supports POSIX class stuff inside a class. Perl gives an error if @@ -3805,8 +4771,9 @@ for (;; ptr++) (cd->external_options & PCRE_JAVASCRIPT_COMPAT) != 0) { *code++ = negate_class? OP_ALLANY : OP_FAIL; - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; break; } @@ -3816,32 +4783,32 @@ for (;; ptr++) should_flip_negation = FALSE; - /* For optimization purposes, we track some properties of the class. - class_has_8bitchar will be non-zero, if the class contains at least one - < 256 character. class_single_char will be 1 if the class contains only - a single character. */ + /* For optimization purposes, we track some properties of the class: + class_has_8bitchar will be non-zero if the class contains at least one < + 256 character; class_one_char will be 1 if the class contains just one + character. */ class_has_8bitchar = 0; - class_single_char = 0; + class_one_char = 0; /* Initialize the 32-char bit map to all zeros. We build the map in a - temporary bit of memory, in case the class contains only 1 character (less - than 256), because in that case the compiled code doesn't use the bit map. - */ + temporary bit of memory, in case the class contains fewer than two + 8-bit characters because in that case the compiled code doesn't use the bit + map. */ memset(classbits, 0, 32 * sizeof(pcre_uint8)); #if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - xclass = FALSE; /* No chars >= 256 */ - class_uchardata = code + LINK_SIZE + 2; /* For UTF-8 items */ - class_uchardata_base = class_uchardata; /* For resetting in pass 1 */ + xclass = FALSE; + class_uchardata = code + LINK_SIZE + 2; /* For XCLASS items */ + class_uchardata_base = class_uchardata; /* Save the start */ #endif /* Process characters until ] is reached. By writing this as a "do" it means that an initial ] is taken as a data character. At the start of the loop, c contains the first byte of the character. */ - if (c != 0) do + if (c != CHAR_NULL) do { const pcre_uchar *oldptr; @@ -3856,10 +4823,12 @@ for (;; ptr++) /* In the pre-compile phase, accumulate the length of any extra data and reset the pointer. This is so that very large classes that contain a zillion > 255 characters no longer overwrite the work space - (which is on the stack). */ + (which is on the stack). We have to remember that there was XCLASS data, + however. */ - if (lengthptr != NULL) + if (lengthptr != NULL && class_uchardata > class_uchardata_base) { + xclass = TRUE; *lengthptr += class_uchardata - class_uchardata_base; class_uchardata = class_uchardata_base; } @@ -3922,24 +4891,58 @@ for (;; ptr++) posix_class = 0; /* When PCRE_UCP is set, some of the POSIX classes are converted to - different escape sequences that use Unicode properties. */ + different escape sequences that use Unicode properties \p or \P. Others + that are not available via \p or \P generate XCL_PROP/XCL_NOTPROP + directly. */ #ifdef SUPPORT_UCP if ((options & PCRE_UCP) != 0) { + unsigned int ptype = 0; int pc = posix_class + ((local_negate)? POSIX_SUBSIZE/2 : 0); + + /* The posix_substitutes table specifies which POSIX classes can be + converted to \p or \P items. */ + if (posix_substitutes[pc] != NULL) { nestptr = tempptr + 1; ptr = posix_substitutes[pc] - 1; continue; } + + /* There are three other classes that generate special property calls + that are recognized only in an XCLASS. */ + + else switch(posix_class) + { + case PC_GRAPH: + ptype = PT_PXGRAPH; + /* Fall through */ + case PC_PRINT: + if (ptype == 0) ptype = PT_PXPRINT; + /* Fall through */ + case PC_PUNCT: + if (ptype == 0) ptype = PT_PXPUNCT; + *class_uchardata++ = local_negate? XCL_NOTPROP : XCL_PROP; + *class_uchardata++ = ptype; + *class_uchardata++ = 0; + ptr = tempptr + 1; + continue; + + /* For all other POSIX classes, no special action is taken in UCP + mode. Fall through to the non_UCP case. */ + + default: + break; + } } #endif - /* In the non-UCP case, we build the bit map for the POSIX class in a - chunk of local store because we may be adding and subtracting from it, - and we don't want to subtract bits that may be in the main map already. - At the end we or the result into the bit map that is being built. */ + /* In the non-UCP case, or when UCP makes no difference, we build the + bit map for the POSIX class in a chunk of local store because we may be + adding and subtracting from it, and we don't want to subtract bits that + may be in the main map already. At the end we or the result into the + bit map that is being built. */ posix_class *= 3; @@ -3961,7 +4964,7 @@ for (;; ptr++) for (c = 0; c < 32; c++) pbits[c] &= ~cbits[c + taboffset]; } - /* Not see if we need to remove any special characters. An option + /* Now see if we need to remove any special characters. An option value of 1 removes vertical space and 2 removes underscore. */ if (tabopt < 0) tabopt = -tabopt; @@ -3977,10 +4980,10 @@ for (;; ptr++) for (c = 0; c < 32; c++) classbits[c] |= pbits[c]; ptr = tempptr + 1; - /* Every class contains at least one < 256 characters. */ + /* Every class contains at least one < 256 character. */ class_has_8bitchar = 1; /* Every class contains at least two characters. */ - class_single_char = 2; + class_one_char = 2; continue; /* End of POSIX syntax handling */ } @@ -3988,23 +4991,24 @@ for (;; ptr++) of the specials, which just set a flag. The sequence \b is a special case. Inside a class (and only there) it is treated as backspace. We assume that other escapes have more than one character in them, so - speculatively set both class_has_8bitchar and class_single_char bigger + speculatively set both class_has_8bitchar and class_one_char bigger than one. Unrecognized escapes fall through and are either treated as literal characters (by default), or are faulted if PCRE_EXTRA is set. */ if (c == CHAR_BACKSLASH) { - c = check_escape(&ptr, errorcodeptr, cd->bracount, options, TRUE); + escape = check_escape(&ptr, &ec, errorcodeptr, cd->bracount, options, + TRUE); if (*errorcodeptr != 0) goto FAILED; - - if (-c == ESC_b) c = CHAR_BS; /* \b is backspace in a class */ - else if (-c == ESC_N) /* \N is not supported in a class */ + if (escape == 0) c = ec; + else if (escape == ESC_b) c = CHAR_BS; /* \b is backspace in a class */ + else if (escape == ESC_N) /* \N is not supported in a class */ { *errorcodeptr = ERR71; goto FAILED; } - else if (-c == ESC_Q) /* Handle start of quoted string */ + else if (escape == ESC_Q) /* Handle start of quoted string */ { if (ptr[1] == CHAR_BACKSLASH && ptr[2] == CHAR_E) { @@ -4013,17 +5017,17 @@ for (;; ptr++) else inescq = TRUE; continue; } - else if (-c == ESC_E) continue; /* Ignore orphan \E */ + else if (escape == ESC_E) continue; /* Ignore orphan \E */ - if (c < 0) + else { register const pcre_uint8 *cbits = cd->cbits; /* Every class contains at least two < 256 characters. */ class_has_8bitchar++; /* Every class contains at least two characters. */ - class_single_char += 2; + class_one_char += 2; - switch (-c) + switch (escape) { #ifdef SUPPORT_UCP case ESC_du: /* These are the values given for \d etc */ @@ -4033,7 +5037,7 @@ for (;; ptr++) case ESC_su: /* of the default ASCII testing. */ case ESC_SU: nestptr = ptr; - ptr = substitutes[-c - ESC_DU] - 1; /* Just before substitute */ + ptr = substitutes[escape - ESC_DU] - 1; /* Just before substitute */ class_has_8bitchar--; /* Undo! */ continue; #endif @@ -4055,196 +5059,42 @@ for (;; ptr++) for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_word]; continue; - /* Perl 5.004 onwards omits VT from \s, but we must preserve it - if it was previously set by something earlier in the character - class. */ + /* Perl 5.004 onwards omitted VT from \s, but restored it at Perl + 5.18. Before PCRE 8.34, we had to preserve the VT bit if it was + previously set by something earlier in the character class. + Luckily, the value of CHAR_VT is 0x0b in both ASCII and EBCDIC, so + we could just adjust the appropriate bit. From PCRE 8.34 we no + longer treat \s and \S specially. */ case ESC_s: - classbits[0] |= cbits[cbit_space]; - classbits[1] |= cbits[cbit_space+1] & ~0x08; - for (c = 2; c < 32; c++) classbits[c] |= cbits[c+cbit_space]; + for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_space]; continue; case ESC_S: should_flip_negation = TRUE; for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_space]; - classbits[1] |= 0x08; /* Perl 5.004 onwards omits VT from \s */ continue; + /* The rest apply in both UCP and non-UCP cases. */ + case ESC_h: - SETBIT(classbits, 0x09); /* VT */ - SETBIT(classbits, 0x20); /* SPACE */ - SETBIT(classbits, 0xa0); /* NSBP */ -#ifndef COMPILE_PCRE8 - xclass = TRUE; - *class_uchardata++ = XCL_SINGLE; - *class_uchardata++ = 0x1680; - *class_uchardata++ = XCL_SINGLE; - *class_uchardata++ = 0x180e; - *class_uchardata++ = XCL_RANGE; - *class_uchardata++ = 0x2000; - *class_uchardata++ = 0x200a; - *class_uchardata++ = XCL_SINGLE; - *class_uchardata++ = 0x202f; - *class_uchardata++ = XCL_SINGLE; - *class_uchardata++ = 0x205f; - *class_uchardata++ = XCL_SINGLE; - *class_uchardata++ = 0x3000; -#elif defined SUPPORT_UTF - if (utf) - { - xclass = TRUE; - *class_uchardata++ = XCL_SINGLE; - class_uchardata += PRIV(ord2utf)(0x1680, class_uchardata); - *class_uchardata++ = XCL_SINGLE; - class_uchardata += PRIV(ord2utf)(0x180e, class_uchardata); - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x2000, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x200a, class_uchardata); - *class_uchardata++ = XCL_SINGLE; - class_uchardata += PRIV(ord2utf)(0x202f, class_uchardata); - *class_uchardata++ = XCL_SINGLE; - class_uchardata += PRIV(ord2utf)(0x205f, class_uchardata); - *class_uchardata++ = XCL_SINGLE; - class_uchardata += PRIV(ord2utf)(0x3000, class_uchardata); - } -#endif + (void)add_list_to_class(classbits, &class_uchardata, options, cd, + PRIV(hspace_list), NOTACHAR); continue; case ESC_H: - for (c = 0; c < 32; c++) - { - int x = 0xff; - switch (c) - { - case 0x09/8: x ^= 1 << (0x09%8); break; - case 0x20/8: x ^= 1 << (0x20%8); break; - case 0xa0/8: x ^= 1 << (0xa0%8); break; - default: break; - } - classbits[c] |= x; - } -#ifndef COMPILE_PCRE8 - xclass = TRUE; - *class_uchardata++ = XCL_RANGE; - *class_uchardata++ = 0x0100; - *class_uchardata++ = 0x167f; - *class_uchardata++ = XCL_RANGE; - *class_uchardata++ = 0x1681; - *class_uchardata++ = 0x180d; - *class_uchardata++ = XCL_RANGE; - *class_uchardata++ = 0x180f; - *class_uchardata++ = 0x1fff; - *class_uchardata++ = XCL_RANGE; - *class_uchardata++ = 0x200b; - *class_uchardata++ = 0x202e; - *class_uchardata++ = XCL_RANGE; - *class_uchardata++ = 0x2030; - *class_uchardata++ = 0x205e; - *class_uchardata++ = XCL_RANGE; - *class_uchardata++ = 0x2060; - *class_uchardata++ = 0x2fff; - *class_uchardata++ = XCL_RANGE; - *class_uchardata++ = 0x3001; -#ifdef SUPPORT_UTF - if (utf) - class_uchardata += PRIV(ord2utf)(0x10ffff, class_uchardata); - else -#endif - *class_uchardata++ = 0xffff; -#elif defined SUPPORT_UTF - if (utf) - { - xclass = TRUE; - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x0100, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x167f, class_uchardata); - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x1681, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x180d, class_uchardata); - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x180f, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x1fff, class_uchardata); - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x200b, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x202e, class_uchardata); - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x2030, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x205e, class_uchardata); - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x2060, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x2fff, class_uchardata); - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x3001, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x10ffff, class_uchardata); - } -#endif + (void)add_not_list_to_class(classbits, &class_uchardata, options, + cd, PRIV(hspace_list)); continue; case ESC_v: - SETBIT(classbits, 0x0a); /* LF */ - SETBIT(classbits, 0x0b); /* VT */ - SETBIT(classbits, 0x0c); /* FF */ - SETBIT(classbits, 0x0d); /* CR */ - SETBIT(classbits, 0x85); /* NEL */ -#ifndef COMPILE_PCRE8 - xclass = TRUE; - *class_uchardata++ = XCL_RANGE; - *class_uchardata++ = 0x2028; - *class_uchardata++ = 0x2029; -#elif defined SUPPORT_UTF - if (utf) - { - xclass = TRUE; - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x2028, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x2029, class_uchardata); - } -#endif + (void)add_list_to_class(classbits, &class_uchardata, options, cd, + PRIV(vspace_list), NOTACHAR); continue; case ESC_V: - for (c = 0; c < 32; c++) - { - int x = 0xff; - switch (c) - { - case 0x0a/8: x ^= 1 << (0x0a%8); - x ^= 1 << (0x0b%8); - x ^= 1 << (0x0c%8); - x ^= 1 << (0x0d%8); - break; - case 0x85/8: x ^= 1 << (0x85%8); break; - default: break; - } - classbits[c] |= x; - } - -#ifndef COMPILE_PCRE8 - xclass = TRUE; - *class_uchardata++ = XCL_RANGE; - *class_uchardata++ = 0x0100; - *class_uchardata++ = 0x2027; - *class_uchardata++ = XCL_RANGE; - *class_uchardata++ = 0x202a; -#ifdef SUPPORT_UTF - if (utf) - class_uchardata += PRIV(ord2utf)(0x10ffff, class_uchardata); - else -#endif - *class_uchardata++ = 0xffff; -#elif defined SUPPORT_UTF - if (utf) - { - xclass = TRUE; - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x0100, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x2027, class_uchardata); - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x202a, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x10ffff, class_uchardata); - } -#endif + (void)add_not_list_to_class(classbits, &class_uchardata, options, + cd, PRIV(vspace_list)); continue; #ifdef SUPPORT_UCP @@ -4252,11 +5102,10 @@ for (;; ptr++) case ESC_P: { BOOL negated; - int pdata; - int ptype = get_ucp(&ptr, &negated, &pdata, errorcodeptr); - if (ptype < 0) goto FAILED; - xclass = TRUE; - *class_uchardata++ = ((-c == ESC_p) != negated)? + unsigned int ptype = 0, pdata = 0; + if (!get_ucp(&ptr, &negated, &ptype, &pdata, errorcodeptr)) + goto FAILED; + *class_uchardata++ = ((escape == ESC_p) != negated)? XCL_PROP : XCL_NOTPROP; *class_uchardata++ = ptype; *class_uchardata++ = pdata; @@ -4275,21 +5124,23 @@ for (;; ptr++) goto FAILED; } class_has_8bitchar--; /* Undo the speculative increase. */ - class_single_char -= 2; /* Undo the speculative increase. */ + class_one_char -= 2; /* Undo the speculative increase. */ c = *ptr; /* Get the final character and fall through */ break; } } - /* Fall through if we have a single character (c >= 0). This may be - greater than 256. */ + /* Fall through if the escape just defined a single character (c >= 0). + This may be greater than 256. */ + escape = 0; + } /* End of backslash handling */ - /* A single character may be followed by '-' to form a range. However, - Perl does not permit ']' to be the end of the range. A '-' character - at the end is treated as a literal. Perl ignores orphaned \E sequences - entirely. The code for handling \Q and \E is messy. */ + /* A character may be followed by '-' to form a range. However, Perl does + not permit ']' to be the end of the range. A '-' character at the end is + treated as a literal. Perl ignores orphaned \E sequences entirely. The + code for handling \Q and \E is messy. */ CHECK_RANGE: while (ptr[1] == CHAR_BACKSLASH && ptr[2] == CHAR_E) @@ -4297,10 +5148,9 @@ for (;; ptr++) inescq = FALSE; ptr += 2; } - oldptr = ptr; - /* Remember \r or \n */ + /* Remember if \r or \n were explicitly used */ if (c == CHAR_CR || c == CHAR_NL) cd->external_flags |= PCRE_HASCRORLF; @@ -4308,7 +5158,7 @@ for (;; ptr++) if (!inescq && ptr[1] == CHAR_MINUS) { - int d; + pcre_uint32 d; ptr += 2; while (*ptr == CHAR_BACKSLASH && ptr[1] == CHAR_E) ptr += 2; @@ -4324,12 +5174,17 @@ for (;; ptr++) break; } - if (*ptr == 0 || (!inescq && *ptr == CHAR_RIGHT_SQUARE_BRACKET)) + /* Minus (hyphen) at the end of a class is treated as a literal, so put + back the pointer and jump to handle the character that preceded it. */ + + if (*ptr == CHAR_NULL || (!inescq && *ptr == CHAR_RIGHT_SQUARE_BRACKET)) { ptr = oldptr; - goto LONE_SINGLE_CHARACTER; + goto CLASS_SINGLE_CHARACTER; } + /* Otherwise, we have a potential range; pick up the next character */ + #ifdef SUPPORT_UTF if (utf) { /* Braces are required because the */ @@ -4339,234 +5194,139 @@ for (;; ptr++) #endif d = *ptr; /* Not UTF-8 mode */ - /* The second part of a range can be a single-character escape, but - not any of the other escapes. Perl 5.6 treats a hyphen as a literal - in such circumstances. */ + /* The second part of a range can be a single-character escape + sequence, but not any of the other escapes. Perl treats a hyphen as a + literal in such circumstances. However, in Perl's warning mode, a + warning is given, so PCRE now faults it as it is almost certainly a + mistake on the user's part. */ - if (!inescq && d == CHAR_BACKSLASH) + if (!inescq) { - d = check_escape(&ptr, errorcodeptr, cd->bracount, options, TRUE); - if (*errorcodeptr != 0) goto FAILED; + if (d == CHAR_BACKSLASH) + { + int descape; + descape = check_escape(&ptr, &d, errorcodeptr, cd->bracount, options, TRUE); + if (*errorcodeptr != 0) goto FAILED; - /* \b is backspace; any other special means the '-' was literal */ + /* 0 means a character was put into d; \b is backspace; any other + special causes an error. */ - if (d < 0) - { - if (d == -ESC_b) d = CHAR_BS; else + if (descape != 0) { - ptr = oldptr; - goto LONE_SINGLE_CHARACTER; /* A few lines below */ + if (descape == ESC_b) d = CHAR_BS; else + { + *errorcodeptr = ERR83; + goto FAILED; + } } } + + /* A hyphen followed by a POSIX class is treated in the same way. */ + + else if (d == CHAR_LEFT_SQUARE_BRACKET && + (ptr[1] == CHAR_COLON || ptr[1] == CHAR_DOT || + ptr[1] == CHAR_EQUALS_SIGN) && + check_posix_syntax(ptr, &tempptr)) + { + *errorcodeptr = ERR83; + goto FAILED; + } } /* Check that the two values are in the correct order. Optimize - one-character ranges */ + one-character ranges. */ if (d < c) { *errorcodeptr = ERR8; goto FAILED; } + if (d == c) goto CLASS_SINGLE_CHARACTER; /* A few lines below */ - if (d == c) goto LONE_SINGLE_CHARACTER; /* A few lines below */ + /* We have found a character range, so single character optimizations + cannot be done anymore. Any value greater than 1 indicates that there + is more than one character. */ - /* Remember \r or \n */ + class_one_char = 2; + /* Remember an explicit \r or \n, and add the range to the class. */ + if (d == CHAR_CR || d == CHAR_NL) cd->external_flags |= PCRE_HASCRORLF; - /* Since we found a character range, single character optimizations - cannot be done anymore. */ - class_single_char = 2; + class_has_8bitchar += + add_to_class(classbits, &class_uchardata, options, cd, c, d); - /* In UTF-8 mode, if the upper limit is > 255, or > 127 for caseless - matching, we have to use an XCLASS with extra data items. Caseless - matching for characters > 127 is available only if UCP support is - available. */ + continue; /* Go get the next char in the class */ + } -#if defined SUPPORT_UTF && !(defined COMPILE_PCRE8) - if ((d > 255) || (utf && ((options & PCRE_CASELESS) != 0 && d > 127))) -#elif defined SUPPORT_UTF - if (utf && (d > 255 || ((options & PCRE_CASELESS) != 0 && d > 127))) -#elif !(defined COMPILE_PCRE8) - if (d > 255) -#endif -#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) - { - xclass = TRUE; + /* Handle a single character - we can get here for a normal non-escape + char, or after \ that introduces a single character or for an apparent + range that isn't. Only the value 1 matters for class_one_char, so don't + increase it if it is already 2 or more ... just in case there's a class + with a zillion characters in it. */ - /* With UCP support, we can find the other case equivalents of - the relevant characters. There may be several ranges. Optimize how - they fit with the basic range. */ + CLASS_SINGLE_CHARACTER: + if (class_one_char < 2) class_one_char++; -#ifdef SUPPORT_UCP -#ifndef COMPILE_PCRE8 - if (utf && (options & PCRE_CASELESS) != 0) -#else - if ((options & PCRE_CASELESS) != 0) -#endif - { - unsigned int occ, ocd; - unsigned int cc = c; - unsigned int origd = d; - while (get_othercase_range(&cc, origd, &occ, &ocd)) - { - if (occ >= (unsigned int)c && - ocd <= (unsigned int)d) - continue; /* Skip embedded ranges */ + /* If class_one_char is 1, we have the first single character in the + class, and there have been no prior ranges, or XCLASS items generated by + escapes. If this is the final character in the class, we can optimize by + turning the item into a 1-character OP_CHAR[I] if it's positive, or + OP_NOT[I] if it's negative. In the positive case, it can cause firstchar + to be set. Otherwise, there can be no first char if this item is first, + whatever repeat count may follow. In the case of reqchar, save the + previous value for reinstating. */ - if (occ < (unsigned int)c && - ocd >= (unsigned int)c - 1) /* Extend the basic range */ - { /* if there is overlap, */ - c = occ; /* noting that if occ < c */ - continue; /* we can't have ocd > d */ - } /* because a subrange is */ - if (ocd > (unsigned int)d && - occ <= (unsigned int)d + 1) /* always shorter than */ - { /* the basic range. */ - d = ocd; - continue; - } + if (class_one_char == 1 && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET) + { + ptr++; + zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; - if (occ == ocd) - { - *class_uchardata++ = XCL_SINGLE; - } - else - { - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(occ, class_uchardata); - } - class_uchardata += PRIV(ord2utf)(ocd, class_uchardata); - } - } -#endif /* SUPPORT_UCP */ - - /* Now record the original range, possibly modified for UCP caseless - overlapping ranges. */ - - *class_uchardata++ = XCL_RANGE; -#ifdef SUPPORT_UTF -#ifndef COMPILE_PCRE8 - if (utf) - { - class_uchardata += PRIV(ord2utf)(c, class_uchardata); - class_uchardata += PRIV(ord2utf)(d, class_uchardata); - } - else - { - *class_uchardata++ = c; - *class_uchardata++ = d; - } -#else - class_uchardata += PRIV(ord2utf)(c, class_uchardata); - class_uchardata += PRIV(ord2utf)(d, class_uchardata); + if (negate_class) + { +#ifdef SUPPORT_UCP + int d; #endif -#else /* SUPPORT_UTF */ - *class_uchardata++ = c; - *class_uchardata++ = d; -#endif /* SUPPORT_UTF */ + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; + zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; - /* With UCP support, we are done. Without UCP support, there is no - caseless matching for UTF characters > 127; we can use the bit map - for the smaller ones. As for 16 bit characters without UTF, we - can still use */ + /* For caseless UTF-8 mode when UCP support is available, check + whether this character has more than one other case. If so, generate + a special OP_NOTPROP item instead of OP_NOTI. */ #ifdef SUPPORT_UCP -#ifndef COMPILE_PCRE8 - if (utf) -#endif - continue; /* With next character in the class */ -#endif /* SUPPORT_UCP */ - -#if defined SUPPORT_UTF && !defined(SUPPORT_UCP) && !(defined COMPILE_PCRE8) - if (utf) + if (utf && (options & PCRE_CASELESS) != 0 && + (d = UCD_CASESET(c)) != 0) { - if ((options & PCRE_CASELESS) == 0 || c > 127) continue; - /* Adjust upper limit and fall through to set up the map */ - d = 127; + *code++ = OP_NOTPROP; + *code++ = PT_CLIST; + *code++ = d; } else - { - if (c > 255) continue; - /* Adjust upper limit and fall through to set up the map */ - d = 255; - } -#elif defined SUPPORT_UTF && !defined(SUPPORT_UCP) - if ((options & PCRE_CASELESS) == 0 || c > 127) continue; - /* Adjust upper limit and fall through to set up the map */ - d = 127; -#else - if (c > 255) continue; - /* Adjust upper limit and fall through to set up the map */ - d = 255; -#endif /* SUPPORT_UTF && !SUPPORT_UCP && !COMPILE_PCRE8 */ - } -#endif /* SUPPORT_UTF || !COMPILE_PCRE8 */ +#endif + /* Char has only one other case, or UCP not available */ - /* We use the bit map for 8 bit mode, or when the characters fall - partially or entirely to [0-255] ([0-127] for UCP) ranges. */ - - class_has_8bitchar = 1; - - /* We can save a bit of time by skipping this in the pre-compile. */ - - if (lengthptr == NULL) for (; c <= d; c++) - { - classbits[c/8] |= (1 << (c&7)); - if ((options & PCRE_CASELESS) != 0) { - int uc = cd->fcc[c]; /* flip case */ - classbits[uc/8] |= (1 << (uc&7)); + *code++ = ((options & PCRE_CASELESS) != 0)? OP_NOTI: OP_NOT; +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf && c > MAX_VALUE_FOR_SINGLE_CHAR) + code += PRIV(ord2utf)(c, code); + else +#endif + *code++ = c; } - } - continue; /* Go get the next char in the class */ - } + /* We are finished with this character class */ - /* Handle a lone single character - we can get here for a normal - non-escape char, or after \ that introduces a single character or for an - apparent range that isn't. */ - - LONE_SINGLE_CHARACTER: - - /* Only the value of 1 matters for class_single_char. */ - - if (class_single_char < 2) class_single_char++; - - /* If class_charcount is 1, we saw precisely one character. As long as - there was no use of \p or \P, in other words, no use of any XCLASS - features, we can optimize. - - The optimization throws away the bit map. We turn the item into a - 1-character OP_CHAR[I] if it's positive, or OP_NOT[I] if it's negative. - In the positive case, it can cause firstchar to be set. Otherwise, there - can be no first char if this item is first, whatever repeat count may - follow. In the case of reqchar, save the previous value for reinstating. */ - - if (class_single_char == 1 && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET) - { - ptr++; - zeroreqchar = reqchar; - - if (negate_class) - { - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; - zerofirstchar = firstchar; - *code++ = ((options & PCRE_CASELESS) != 0)? OP_NOTI: OP_NOT; -#ifdef SUPPORT_UTF - if (utf && c > MAX_VALUE_FOR_SINGLE_CHAR) - code += PRIV(ord2utf)(c, code); - else -#endif - *code++ = c; - goto NOT_CHAR; + goto END_CLASS; } /* For a single, positive character, get the value into mcbuffer, and then we can handle this with the normal one-character code. */ -#ifdef SUPPORT_UTF +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 if (utf && c > MAX_VALUE_FOR_SINGLE_CHAR) mclength = PRIV(ord2utf)(c, mcbuffer); else @@ -4578,89 +5338,51 @@ for (;; ptr++) goto ONE_CHAR; } /* End of 1-char optimization */ - /* Handle a character that cannot go in the bit map. */ + /* There is more than one character in the class, or an XCLASS item + has been generated. Add this character to the class. */ -#if defined SUPPORT_UTF && !(defined COMPILE_PCRE8) - if ((c > 255) || (utf && ((options & PCRE_CASELESS) != 0 && c > 127))) -#elif defined SUPPORT_UTF - if (utf && (c > 255 || ((options & PCRE_CASELESS) != 0 && c > 127))) -#elif !(defined COMPILE_PCRE8) - if (c > 255) -#endif - -#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) - { - xclass = TRUE; - *class_uchardata++ = XCL_SINGLE; -#ifdef SUPPORT_UTF -#ifndef COMPILE_PCRE8 - /* In non 8 bit mode, we can get here even if we are not in UTF mode. */ - if (!utf) - *class_uchardata++ = c; - else -#endif - class_uchardata += PRIV(ord2utf)(c, class_uchardata); -#else /* SUPPORT_UTF */ - *class_uchardata++ = c; -#endif /* SUPPORT_UTF */ - -#ifdef SUPPORT_UCP -#ifdef COMPILE_PCRE8 - if ((options & PCRE_CASELESS) != 0) -#else - /* In non 8 bit mode, we can get here even if we are not in UTF mode. */ - if (utf && (options & PCRE_CASELESS) != 0) -#endif - { - unsigned int othercase; - if ((int)(othercase = UCD_OTHERCASE(c)) != c) - { - *class_uchardata++ = XCL_SINGLE; - class_uchardata += PRIV(ord2utf)(othercase, class_uchardata); - } - } -#endif /* SUPPORT_UCP */ - - } - else -#endif /* SUPPORT_UTF || COMPILE_PCRE16 */ - - /* Handle a single-byte character */ - { - class_has_8bitchar = 1; - classbits[c/8] |= (1 << (c&7)); - if ((options & PCRE_CASELESS) != 0) - { - c = cd->fcc[c]; /* flip case */ - classbits[c/8] |= (1 << (c&7)); - } - } + class_has_8bitchar += + add_to_class(classbits, &class_uchardata, options, cd, c, c); } /* Loop until ']' reached. This "while" is the end of the "do" far above. If we are at the end of an internal nested string, revert to the outer string. */ - while (((c = *(++ptr)) != 0 || + while (((c = *(++ptr)) != CHAR_NULL || (nestptr != NULL && - (ptr = nestptr, nestptr = NULL, c = *(++ptr)) != 0)) && + (ptr = nestptr, nestptr = NULL, c = *(++ptr)) != CHAR_NULL)) && (c != CHAR_RIGHT_SQUARE_BRACKET || inescq)); /* Check for missing terminating ']' */ - if (c == 0) + if (c == CHAR_NULL) { *errorcodeptr = ERR6; goto FAILED; } + /* We will need an XCLASS if data has been placed in class_uchardata. In + the second phase this is a sufficient test. However, in the pre-compile + phase, class_uchardata gets emptied to prevent workspace overflow, so it + only if the very last character in the class needs XCLASS will it contain + anything at this point. For this reason, xclass gets set TRUE above when + uchar_classdata is emptied, and that's why this code is the way it is here + instead of just doing a test on class_uchardata below. */ + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + if (class_uchardata > class_uchardata_base) xclass = TRUE; +#endif + /* If this is the first thing in the branch, there can be no first char setting, whatever the repeat count. Any reqchar setting must remain unchanged after any kind of repeat. */ - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; /* If there are characters with values > 255, we have to compile an extended class, with its own opcode, unless there was a negated special @@ -4716,7 +5438,8 @@ for (;; ptr++) memcpy(code, classbits, 32); } code += 32 / sizeof(pcre_uchar); - NOT_CHAR: + + END_CLASS: break; @@ -4754,7 +5477,9 @@ for (;; ptr++) if (repeat_min == 0) { firstchar = zerofirstchar; /* Adjust for zero repeat */ + firstcharflags = zerofirstcharflags; reqchar = zeroreqchar; /* Ditto */ + reqcharflags = zeroreqcharflags; } /* Remember whether this is a variable length repeat */ @@ -4769,6 +5494,34 @@ for (;; ptr++) tempcode = previous; + /* Before checking for a possessive quantifier, we must skip over + whitespace and comments in extended mode because Perl allows white space at + this point. */ + + if ((options & PCRE_EXTENDED) != 0) + { + const pcre_uchar *p = ptr + 1; + for (;;) + { + while (MAX_255(*p) && (cd->ctypes[*p] & ctype_space) != 0) p++; + if (*p != CHAR_NUMBER_SIGN) break; + p++; + while (*p != CHAR_NULL) + { + if (IS_NEWLINE(p)) /* For non-fixed-length newline cases, */ + { /* IS_NEWLINE sets cd->nllen. */ + p += cd->nllen; + break; + } + p++; +#ifdef SUPPORT_UTF + if (utf) FORWARDCHAR(p); +#endif + } /* Loop for comment characters */ + } /* Loop for multiple comments */ + ptr = p - 1; /* Character before the next significant one. */ + } + /* If the next character is '+', we have a possessive quantifier. This implies greediness, whatever the setting of the PCRE_UNGREEDY option. If the next character is '?' this is a minimizing repeat, by default, @@ -4840,7 +5593,7 @@ for (;; ptr++) hold the length of the character in bytes, plus UTF_LENGTH to flag that it's a length rather than a small character. */ -#ifdef SUPPORT_UTF +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 if (utf && NOT_FIRSTCHAR(code[-1])) { pcre_uchar *lastchar = code - 1; @@ -4857,22 +5610,12 @@ for (;; ptr++) { c = code[-1]; if (*previous <= OP_CHARI && repeat_min > 1) - reqchar = c | req_caseopt | cd->req_varyopt; + { + reqchar = c; + reqcharflags = req_caseopt | cd->req_varyopt; + } } - /* If the repetition is unlimited, it pays to see if the next thing on - the line is something that cannot possibly match this character. If so, - automatically possessifying this item gains some performance in the case - where the match fails. */ - - if (!possessive_quantifier && - repeat_max < 0 && - check_auto_possessive(previous, utf, ptr + 1, options, cd)) - { - repeat_type = 0; /* Force greedy */ - possessive_quantifier = TRUE; - } - goto OUTPUT_SINGLE_REPEAT; /* Code shared with single character types */ } @@ -4890,14 +5633,6 @@ for (;; ptr++) op_type = OP_TYPESTAR - OP_STAR; /* Use type opcodes */ c = *previous; - if (!possessive_quantifier && - repeat_max < 0 && - check_auto_possessive(previous, utf, ptr + 1, options, cd)) - { - repeat_type = 0; /* Force greedy */ - possessive_quantifier = TRUE; - } - OUTPUT_SINGLE_REPEAT: if (*previous == OP_PROP || *previous == OP_NOTPROP) { @@ -4914,16 +5649,6 @@ for (;; ptr++) if (repeat_max == 0) goto END_REPEAT; - /*--------------------------------------------------------------------*/ - /* This code is obsolete from release 8.00; the restriction was finally - removed: */ - - /* All real repeats make it impossible to handle partial matching (maybe - one day we will be able to remove this restriction). */ - - /* if (repeat_max != 1) cd->external_flags |= PCRE_NOPARTIAL; */ - /*--------------------------------------------------------------------*/ - /* Combine the op_type with the repeat_type */ repeat_type += op_type; @@ -4976,7 +5701,7 @@ for (;; ptr++) if (repeat_max < 0) { -#ifdef SUPPORT_UTF +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 if (utf && (c & UTF_LENGTH) != 0) { memcpy(code, utf_chars, IN_UCHARS(c & 7)); @@ -5001,7 +5726,7 @@ for (;; ptr++) else if (repeat_max != repeat_min) { -#ifdef SUPPORT_UTF +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 if (utf && (c & UTF_LENGTH) != 0) { memcpy(code, utf_chars, IN_UCHARS(c & 7)); @@ -5031,7 +5756,7 @@ for (;; ptr++) /* The character or character type itself comes last in all cases. */ -#ifdef SUPPORT_UTF +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 if (utf && (c & UTF_LENGTH) != 0) { memcpy(code, utf_chars, IN_UCHARS(c & 7)); @@ -5056,13 +5781,12 @@ for (;; ptr++) /* If previous was a character class or a back reference, we put the repeat stuff after it, but just skip the item if the repeat was {0,0}. */ - else if (*previous == OP_CLASS || - *previous == OP_NCLASS || + else if (*previous == OP_CLASS || *previous == OP_NCLASS || #if defined SUPPORT_UTF || !defined COMPILE_PCRE8 *previous == OP_XCLASS || #endif - *previous == OP_REF || - *previous == OP_REFI) + *previous == OP_REF || *previous == OP_REFI || + *previous == OP_DNREF || *previous == OP_DNREFI) { if (repeat_max == 0) { @@ -5070,16 +5794,6 @@ for (;; ptr++) goto END_REPEAT; } - /*--------------------------------------------------------------------*/ - /* This code is obsolete from release 8.00; the restriction was finally - removed: */ - - /* All real repeats make it impossible to handle partial matching (maybe - one day we will be able to remove this restriction). */ - - /* if (repeat_max != 1) cd->external_flags |= PCRE_NOPARTIAL; */ - /*--------------------------------------------------------------------*/ - if (repeat_min == 0 && repeat_max == -1) *code++ = OP_CRSTAR + repeat_type; else if (repeat_min == 1 && repeat_max == -1) @@ -5100,8 +5814,9 @@ for (;; ptr++) opcodes such as BRA and CBRA, as this is the place where they get converted into the more special varieties such as BRAPOS and SBRA. A test for >= OP_ASSERT and <= OP_COND includes ASSERT, ASSERT_NOT, ASSERTBACK, - ASSERTBACK_NOT, ONCE, BRA, CBRA, and COND. Originally, PCRE did not allow - repetition of assertions, but now it does, for Perl compatibility. */ + ASSERTBACK_NOT, ONCE, ONCE_NC, BRA, BRAPOS, CBRA, CBRAPOS, and COND. + Originally, PCRE did not allow repetition of assertions, but now it does, + for Perl compatibility. */ else if (*previous >= OP_ASSERT && *previous <= OP_COND) { @@ -5119,7 +5834,7 @@ for (;; ptr++) /* There is no sense in actually repeating assertions. The only potential use of repetition is in cases when the assertion is optional. Therefore, if the minimum is greater than zero, just ignore the repeat. If the - maximum is not not zero or one, set it to 1. */ + maximum is not zero or one, set it to 1. */ if (*previous < OP_ONCE) /* Assertion */ { @@ -5239,7 +5954,11 @@ for (;; ptr++) else { - if (groupsetfirstchar && reqchar < 0) reqchar = firstchar; + if (groupsetfirstchar && reqcharflags < 0) + { + reqchar = firstchar; + reqcharflags = firstcharflags; + } for (i = 1; i < repeat_min; i++) { @@ -5418,7 +6137,7 @@ for (;; ptr++) pcre_uchar *scode = bracode; do { - if (could_be_empty_branch(scode, ketcode, utf, cd)) + if (could_be_empty_branch(scode, ketcode, utf, cd, NULL)) { *bracode += OP_SBRA - OP_BRA; break; @@ -5488,43 +6207,105 @@ for (;; ptr++) goto FAILED; } - /* If the character following a repeat is '+', or if certain optimization - tests above succeeded, possessive_quantifier is TRUE. For some opcodes, - there are special alternative opcodes for this case. For anything else, we - wrap the entire repeated item inside OP_ONCE brackets. Logically, the '+' - notation is just syntactic sugar, taken from Sun's Java package, but the - special opcodes can optimize it. + /* If the character following a repeat is '+', possessive_quantifier is + TRUE. For some opcodes, there are special alternative opcodes for this + case. For anything else, we wrap the entire repeated item inside OP_ONCE + brackets. Logically, the '+' notation is just syntactic sugar, taken from + Sun's Java package, but the special opcodes can optimize it. Some (but not all) possessively repeated subpatterns have already been completely handled in the code just above. For them, possessive_quantifier - is always FALSE at this stage. + is always FALSE at this stage. Note that the repeated item starts at + tempcode, not at previous, which might be the first part of a string whose + (former) last char we repeated. */ - Note that the repeated item starts at tempcode, not at previous, which - might be the first part of a string whose (former) last char we repeated. - - Possessifying an 'exact' quantifier has no effect, so we can ignore it. But - an 'upto' may follow. We skip over an 'exact' item, and then test the - length of what remains before proceeding. */ - if (possessive_quantifier) { int len; - if (*tempcode == OP_TYPEEXACT) + /* Possessifying an EXACT quantifier has no effect, so we can ignore it. + However, QUERY, STAR, or UPTO may follow (for quantifiers such as {5,6}, + {5,}, or {5,10}). We skip over an EXACT item; if the length of what + remains is greater than zero, there's a further opcode that can be + handled. If not, do nothing, leaving the EXACT alone. */ + + switch(*tempcode) + { + case OP_TYPEEXACT: tempcode += PRIV(OP_lengths)[*tempcode] + ((tempcode[1 + IMM2_SIZE] == OP_PROP || tempcode[1 + IMM2_SIZE] == OP_NOTPROP)? 2 : 0); + break; - else if (*tempcode == OP_EXACT || *tempcode == OP_NOTEXACT) - { + /* CHAR opcodes are used for exacts whose count is 1. */ + + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + case OP_EXACT: + case OP_EXACTI: + case OP_NOTEXACT: + case OP_NOTEXACTI: tempcode += PRIV(OP_lengths)[*tempcode]; #ifdef SUPPORT_UTF if (utf && HAS_EXTRALEN(tempcode[-1])) tempcode += GET_EXTRALEN(tempcode[-1]); #endif + break; + + /* For the class opcodes, the repeat operator appears at the end; + adjust tempcode to point to it. */ + + case OP_CLASS: + case OP_NCLASS: + tempcode += 1 + 32/sizeof(pcre_uchar); + break; + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + tempcode += GET(tempcode, 1); + break; +#endif } + /* If tempcode is equal to code (which points to the end of the repeated + item), it means we have skipped an EXACT item but there is no following + QUERY, STAR, or UPTO; the value of len will be 0, and we do nothing. In + all other cases, tempcode will be pointing to the repeat opcode, and will + be less than code, so the value of len will be greater than 0. */ + len = (int)(code - tempcode); + if (len > 0) + { + unsigned int repcode = *tempcode; + + /* There is a table for possessifying opcodes, all of which are less + than OP_CALLOUT. A zero entry means there is no possessified version. + */ + + if (repcode < OP_CALLOUT && opcode_possessify[repcode] > 0) + *tempcode = opcode_possessify[repcode]; + + /* For opcode without a special possessified version, wrap the item in + ONCE brackets. Because we are moving code along, we must ensure that any + pending recursive references are updated. */ + + else + { + *code = OP_END; + adjust_recurse(tempcode, 1 + LINK_SIZE, utf, cd, save_hwm); + memmove(tempcode + 1 + LINK_SIZE, tempcode, IN_UCHARS(len)); + code += 1 + LINK_SIZE; + len += 1 + LINK_SIZE; + tempcode[0] = OP_ONCE; + *code++ = OP_KET; + PUTINC(code, 0, len); + PUT(tempcode, 1, len); + } + } + +#ifdef NEVER if (len > 0) switch (*tempcode) { case OP_STAR: *tempcode = OP_POSSTAR; break; @@ -5552,6 +6333,11 @@ for (;; ptr++) case OP_TYPEQUERY: *tempcode = OP_TYPEPOSQUERY; break; case OP_TYPEUPTO: *tempcode = OP_TYPEPOSUPTO; break; + case OP_CRSTAR: *tempcode = OP_CRPOSSTAR; break; + case OP_CRPLUS: *tempcode = OP_CRPOSPLUS; break; + case OP_CRQUERY: *tempcode = OP_CRPOSQUERY; break; + case OP_CRRANGE: *tempcode = OP_CRPOSRANGE; break; + /* Because we are moving code along, we must ensure that any pending recursive references are updated. */ @@ -5567,6 +6353,7 @@ for (;; ptr++) PUT(tempcode, 1, len); break; } +#endif } /* In all case we no longer have a previous item. We also set the @@ -5614,9 +6401,9 @@ for (;; ptr++) if (*ptr == CHAR_COLON) { arg = ++ptr; - while (*ptr != 0 && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; + while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; arglen = (int)(ptr - arg); - if (arglen > (int)MAX_MARK) + if ((unsigned int)arglen > MAX_MARK) { *errorcodeptr = ERR75; goto FAILED; @@ -5636,6 +6423,8 @@ for (;; ptr++) if (namelen == verbs[i].len && STRNCMP_UC_C8(name, vn, namelen) == 0) { + int setverb; + /* Check for open captures before ACCEPT and convert it to ASSERT_ACCEPT if in an assertion. */ @@ -5653,10 +6442,11 @@ for (;; ptr++) *code++ = OP_CLOSE; PUT2INC(code, 0, oc->number); } - *code++ = (cd->assert_depth > 0)? OP_ASSERT_ACCEPT : OP_ACCEPT; + setverb = *code++ = + (cd->assert_depth > 0)? OP_ASSERT_ACCEPT : OP_ACCEPT; /* Do not set firstchar after *ACCEPT */ - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; } /* Handle other cases with/without an argument */ @@ -5668,8 +6458,7 @@ for (;; ptr++) *errorcodeptr = ERR66; goto FAILED; } - *code = verbs[i].op; - if (*code++ == OP_THEN) cd->external_flags |= PCRE_HASTHEN; + setverb = *code++ = verbs[i].op; } else @@ -5679,14 +6468,28 @@ for (;; ptr++) *errorcodeptr = ERR59; goto FAILED; } - *code = verbs[i].op_arg; - if (*code++ == OP_THEN_ARG) cd->external_flags |= PCRE_HASTHEN; + setverb = *code++ = verbs[i].op_arg; *code++ = arglen; memcpy(code, arg, IN_UCHARS(arglen)); code += arglen; *code++ = 0; } + switch (setverb) + { + case OP_THEN: + case OP_THEN_ARG: + cd->external_flags |= PCRE_HASTHEN; + break; + + case OP_PRUNE: + case OP_PRUNE_ARG: + case OP_SKIP: + case OP_SKIP_ARG: + cd->had_pruneorskip = TRUE; + break; + } + break; /* Found verb, exit loop */ } @@ -5712,8 +6515,8 @@ for (;; ptr++) { case CHAR_NUMBER_SIGN: /* Comment; skip to ket */ ptr++; - while (*ptr != 0 && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; - if (*ptr == 0) + while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; + if (*ptr == CHAR_NULL) { *errorcodeptr = ERR18; goto FAILED; @@ -5736,30 +6539,44 @@ for (;; ptr++) /* ------------------------------------------------------------ */ case CHAR_LEFT_PARENTHESIS: bravalue = OP_COND; /* Conditional group */ + tempptr = ptr; /* A condition can be an assertion, a number (referring to a numbered - group), a name (referring to a named group), or 'R', referring to - recursion. R and R&name are also permitted for recursion tests. + group's having been set), a name (referring to a named group), or 'R', + referring to recursion. R and R&name are also permitted for + recursion tests. - There are several syntaxes for testing a named group: (?(name)) is used - by Python; Perl 5.10 onwards uses (?() or (?('name')). + There are ways of testing a named group: (?(name)) is used by Python; + Perl 5.10 onwards uses (?() or (?('name')). - There are two unfortunate ambiguities, caused by history. (a) 'R' can - be the recursive thing or the name 'R' (and similarly for 'R' followed - by digits), and (b) a number could be a name that consists of digits. - In both cases, we look for a name first; if not found, we try the other - cases. */ + There is one unfortunate ambiguity, caused by history. 'R' can be the + recursive thing or the name 'R' (and similarly for 'R' followed by + digits). We look for a name first; if not found, we try the other case. + For compatibility with auto-callouts, we allow a callout to be + specified before a condition that is an assertion. First, check for the + syntax of a callout; if found, adjust the temporary pointer that is + used to check for an assertion condition. That's all that is needed! */ + + if (ptr[1] == CHAR_QUESTION_MARK && ptr[2] == CHAR_C) + { + for (i = 3;; i++) if (!IS_DIGIT(ptr[i])) break; + if (ptr[i] == CHAR_RIGHT_PARENTHESIS) + tempptr += i + 1; + } + /* For conditions that are assertions, check the syntax, and then exit the switch. This will take control down to where bracketed groups, including assertions, are processed. */ - if (ptr[1] == CHAR_QUESTION_MARK && (ptr[2] == CHAR_EQUALS_SIGN || - ptr[2] == CHAR_EXCLAMATION_MARK || ptr[2] == CHAR_LESS_THAN_SIGN)) + if (tempptr[1] == CHAR_QUESTION_MARK && + (tempptr[2] == CHAR_EQUALS_SIGN || + tempptr[2] == CHAR_EXCLAMATION_MARK || + tempptr[2] == CHAR_LESS_THAN_SIGN)) break; - /* Most other conditions use OP_CREF (a couple change to OP_RREF - below), and all need to skip 1+IMM2_SIZE bytes at the start of the group. */ + /* Other conditions use OP_CREF/OP_DNCREF/OP_RREF/OP_DNRREF, and all + need to skip at least 1+IMM2_SIZE bytes at the start of the group. */ code[1+LINK_SIZE] = OP_CREF; skipbytes = 1+IMM2_SIZE; @@ -5767,7 +6584,8 @@ for (;; ptr++) /* Check for a test for recursion in a named group. */ - if (ptr[1] == CHAR_R && ptr[2] == CHAR_AMPERSAND) + ptr++; + if (*ptr == CHAR_R && ptr[1] == CHAR_AMPERSAND) { terminator = -1; ptr += 2; @@ -5775,50 +6593,71 @@ for (;; ptr++) } /* Check for a test for a named group's having been set, using the Perl - syntax (?() or (?('name') */ + syntax (?() or (?('name'), and also allow for the original PCRE + syntax of (?(name) or for (?(+n), (?(-n), and just (?(n). */ - else if (ptr[1] == CHAR_LESS_THAN_SIGN) + else if (*ptr == CHAR_LESS_THAN_SIGN) { terminator = CHAR_GREATER_THAN_SIGN; ptr++; } - else if (ptr[1] == CHAR_APOSTROPHE) + else if (*ptr == CHAR_APOSTROPHE) { terminator = CHAR_APOSTROPHE; ptr++; } else { - terminator = 0; - if (ptr[1] == CHAR_MINUS || ptr[1] == CHAR_PLUS) refsign = *(++ptr); + terminator = CHAR_NULL; + if (*ptr == CHAR_MINUS || *ptr == CHAR_PLUS) refsign = *ptr++; + else if (IS_DIGIT(*ptr)) refsign = 0; } - /* We now expect to read a name; any thing else is an error */ + /* Handle a number */ - if (!MAX_255(ptr[1]) || (cd->ctypes[ptr[1]] & ctype_word) == 0) + if (refsign >= 0) { - ptr += 1; /* To get the right offset */ - *errorcodeptr = ERR28; - goto FAILED; + recno = 0; + while (IS_DIGIT(*ptr)) + { + recno = recno * 10 + (int)(*ptr - CHAR_0); + ptr++; + } } - /* Read the name, but also get it as a number if it's all digits */ + /* Otherwise we expect to read a name; anything else is an error. When + a name is one of a number of duplicates, a different opcode is used and + it needs more memory. Unfortunately we cannot tell whether a name is a + duplicate in the first pass, so we have to allow for more memory. */ - recno = 0; - name = ++ptr; - while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) + else { - if (recno >= 0) - recno = (IS_DIGIT(*ptr))? recno * 10 + *ptr - CHAR_0 : -1; - ptr++; + if (IS_DIGIT(*ptr)) + { + *errorcodeptr = ERR84; + goto FAILED; + } + if (!MAX_255(*ptr) || (cd->ctypes[*ptr] & ctype_word) == 0) + { + *errorcodeptr = ERR28; /* Assertion expected */ + goto FAILED; + } + name = ptr++; + while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) + { + ptr++; + } + namelen = (int)(ptr - name); + if (lengthptr != NULL) *lengthptr += IMM2_SIZE; } - namelen = (int)(ptr - name); - if ((terminator > 0 && *ptr++ != terminator) || + /* Check the terminator */ + + if ((terminator > 0 && *ptr++ != (pcre_uchar)terminator) || *ptr++ != CHAR_RIGHT_PARENTHESIS) { - ptr--; /* Error offset */ - *errorcodeptr = ERR26; + ptr--; /* Error offset */ + *errorcodeptr = ERR26; /* Malformed number or name */ goto FAILED; } @@ -5827,18 +6666,18 @@ for (;; ptr++) if (lengthptr != NULL) break; /* In the real compile we do the work of looking for the actual - reference. If the string started with "+" or "-" we require the rest to - be digits, in which case recno will be set. */ + reference. If refsign is not negative, it means we have a number in + recno. */ - if (refsign > 0) + if (refsign >= 0) { if (recno <= 0) { - *errorcodeptr = ERR58; + *errorcodeptr = ERR35; goto FAILED; } - recno = (refsign == CHAR_MINUS)? - cd->bracount - recno + 1 : recno +cd->bracount; + if (refsign != 0) recno = (refsign == CHAR_MINUS)? + cd->bracount - recno + 1 : recno + cd->bracount; if (recno <= 0 || recno > cd->final_bracount) { *errorcodeptr = ERR15; @@ -5848,11 +6687,7 @@ for (;; ptr++) break; } - /* Otherwise (did not start with "+" or "-"), start by looking for the - name. If we find a name, add one to the opcode to change OP_CREF or - OP_RREF into OP_NCREF or OP_NRREF. These behave exactly the same, - except they record that the reference was originally to a name. The - information is used to check duplicate names. */ + /* Otherwise look for the name. */ slot = cd->name_table; for (i = 0; i < cd->names_found; i++) @@ -5861,32 +6696,43 @@ for (;; ptr++) slot += cd->name_entry_size; } - /* Found a previous named subpattern */ + /* Found the named subpattern. If the name is duplicated, add one to + the opcode to change CREF/RREF into DNCREF/DNRREF and insert + appropriate data values. Otherwise, just insert the unique subpattern + number. */ if (i < cd->names_found) { - recno = GET2(slot, 0); - PUT2(code, 2+LINK_SIZE, recno); - code[1+LINK_SIZE]++; + int offset = i++; + int count = 1; + recno = GET2(slot, 0); /* Number from first found */ + for (; i < cd->names_found; i++) + { + slot += cd->name_entry_size; + if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) != 0) break; + count++; + } + if (count > 1) + { + PUT2(code, 2+LINK_SIZE, offset); + PUT2(code, 2+LINK_SIZE+IMM2_SIZE, count); + skipbytes += IMM2_SIZE; + code[1+LINK_SIZE]++; + } + else /* Not a duplicated name */ + { + PUT2(code, 2+LINK_SIZE, recno); + } } - /* Search the pattern for a forward reference */ + /* If terminator == CHAR_NULL it means that the name followed directly + after the opening parenthesis [e.g. (?(abc)...] and in this case there + are some further alternatives to try. For the cases where terminator != + CHAR_NULL [things like (?(... or (?('name')... or (?(R&name)... ] + we have now checked all the possibilities, so give an error. */ - else if ((i = find_parens(cd, name, namelen, - (options & PCRE_EXTENDED) != 0, utf)) > 0) + else if (terminator != CHAR_NULL) { - PUT2(code, 2+LINK_SIZE, i); - code[1+LINK_SIZE]++; - } - - /* If terminator == 0 it means that the name followed directly after - the opening parenthesis [e.g. (?(abc)...] and in this case there are - some further alternatives to try. For the cases where terminator != 0 - [things like (?(... or (?('name')... or (?(R&name)... ] we have - now checked all the possibilities, so give an error. */ - - else if (terminator != 0) - { *errorcodeptr = ERR15; goto FAILED; } @@ -5920,19 +6766,11 @@ for (;; ptr++) skipbytes = 1; } - /* Check for the "name" actually being a subpattern number. We are - in the second pass here, so final_bracount is set. */ + /* Reference to an unidentified subpattern. */ - else if (recno > 0 && recno <= cd->final_bracount) - { - PUT2(code, 2+LINK_SIZE, recno); - } - - /* Either an unidentified subpattern, or a reference to (?(0) */ - else { - *errorcodeptr = (recno == 0)? ERR35: ERR15; + *errorcodeptr = ERR15; goto FAILED; } break; @@ -5945,11 +6783,18 @@ for (;; ptr++) ptr++; break; + /* Optimize (?!) to (*FAIL) unless it is quantified - which is a weird + thing to do, but Perl allows all assertions to be quantified, and when + they contain capturing parentheses there may be a potential use for + this feature. Not that that applies to a quantified (?!) but we allow + it for uniformity. */ /* ------------------------------------------------------------ */ case CHAR_EXCLAMATION_MARK: /* Negative lookahead */ ptr++; - if (*ptr == CHAR_RIGHT_PARENTHESIS) /* Optimize (?!) */ + if (*ptr == CHAR_RIGHT_PARENTHESIS && ptr[1] != CHAR_ASTERISK && + ptr[1] != CHAR_PLUS && ptr[1] != CHAR_QUESTION_MARK && + (ptr[1] != CHAR_LEFT_CURLY_BRACKET || !is_counted_repeat(ptr+2))) { *code++ = OP_FAIL; previous = NULL; @@ -6042,124 +6887,110 @@ for (;; ptr++) /* ------------------------------------------------------------ */ DEFINE_NAME: /* Come here from (?< handling */ case CHAR_APOSTROPHE: + terminator = (*ptr == CHAR_LESS_THAN_SIGN)? + CHAR_GREATER_THAN_SIGN : CHAR_APOSTROPHE; + name = ++ptr; + if (IS_DIGIT(*ptr)) { - terminator = (*ptr == CHAR_LESS_THAN_SIGN)? - CHAR_GREATER_THAN_SIGN : CHAR_APOSTROPHE; - name = ++ptr; + *errorcodeptr = ERR84; /* Group name must start with non-digit */ + goto FAILED; + } + while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) ptr++; + namelen = (int)(ptr - name); - while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) ptr++; - namelen = (int)(ptr - name); + /* In the pre-compile phase, do a syntax check, remember the longest + name, and then remember the group in a vector, expanding it if + necessary. Duplicates for the same number are skipped; other duplicates + are checked for validity. In the actual compile, there is nothing to + do. */ - /* In the pre-compile phase, just do a syntax check. */ + if (lengthptr != NULL) + { + named_group *ng; + pcre_uint32 number = cd->bracount + 1; - if (lengthptr != NULL) + if (*ptr != (pcre_uchar)terminator) { - if (*ptr != terminator) + *errorcodeptr = ERR42; + goto FAILED; + } + + if (cd->names_found >= MAX_NAME_COUNT) + { + *errorcodeptr = ERR49; + goto FAILED; + } + + if (namelen + IMM2_SIZE + 1 > cd->name_entry_size) + { + cd->name_entry_size = namelen + IMM2_SIZE + 1; + if (namelen > MAX_NAME_SIZE) { - *errorcodeptr = ERR42; + *errorcodeptr = ERR48; goto FAILED; } - if (cd->names_found >= MAX_NAME_COUNT) + } + + /* Scan the list to check for duplicates. For duplicate names, if the + number is the same, break the loop, which causes the name to be + discarded; otherwise, if DUPNAMES is not set, give an error. + If it is set, allow the name with a different number, but continue + scanning in case this is a duplicate with the same number. For + non-duplicate names, give an error if the number is duplicated. */ + + ng = cd->named_groups; + for (i = 0; i < cd->names_found; i++, ng++) + { + if (namelen == ng->length && + STRNCMP_UC_UC(name, ng->name, namelen) == 0) { - *errorcodeptr = ERR49; - goto FAILED; - } - if (namelen + IMM2_SIZE + 1 > cd->name_entry_size) - { - cd->name_entry_size = namelen + IMM2_SIZE + 1; - if (namelen > MAX_NAME_SIZE) + if (ng->number == number) break; + if ((options & PCRE_DUPNAMES) == 0) { - *errorcodeptr = ERR48; + *errorcodeptr = ERR43; goto FAILED; } + cd->dupnames = TRUE; /* Duplicate names exist */ } + else if (ng->number == number) + { + *errorcodeptr = ERR65; + goto FAILED; + } } - /* In the real compile, create the entry in the table, maintaining - alphabetical order. Duplicate names for different numbers are - permitted only if PCRE_DUPNAMES is set. Duplicate names for the same - number are always OK. (An existing number can be re-used if (?| - appears in the pattern.) In either event, a duplicate name results in - a duplicate entry in the table, even if the number is the same. This - is because the number of names, and hence the table size, is computed - in the pre-compile, and it affects various numbers and pointers which - would all have to be modified, and the compiled code moved down, if - duplicates with the same number were omitted from the table. This - doesn't seem worth the hassle. However, *different* names for the - same number are not permitted. */ - - else + if (i >= cd->names_found) /* Not a duplicate with same number */ { - BOOL dupname = FALSE; - slot = cd->name_table; + /* Increase the list size if necessary */ - for (i = 0; i < cd->names_found; i++) + if (cd->names_found >= cd->named_group_list_size) { - int crc = memcmp(name, slot+IMM2_SIZE, IN_UCHARS(namelen)); - if (crc == 0) - { - if (slot[IMM2_SIZE+namelen] == 0) - { - if (GET2(slot, 0) != cd->bracount + 1 && - (options & PCRE_DUPNAMES) == 0) - { - *errorcodeptr = ERR43; - goto FAILED; - } - else dupname = TRUE; - } - else crc = -1; /* Current name is a substring */ - } + int newsize = cd->named_group_list_size * 2; + named_group *newspace = (PUBL(malloc)) + (newsize * sizeof(named_group)); - /* Make space in the table and break the loop for an earlier - name. For a duplicate or later name, carry on. We do this for - duplicates so that in the simple case (when ?(| is not used) they - are in order of their numbers. */ - - if (crc < 0) + if (newspace == NULL) { - memmove(slot + cd->name_entry_size, slot, - IN_UCHARS((cd->names_found - i) * cd->name_entry_size)); - break; + *errorcodeptr = ERR21; + goto FAILED; } - /* Continue the loop for a later or duplicate name */ - - slot += cd->name_entry_size; + memcpy(newspace, cd->named_groups, + cd->named_group_list_size * sizeof(named_group)); + if (cd->named_group_list_size > NAMED_GROUP_LIST_SIZE) + (PUBL(free))((void *)cd->named_groups); + cd->named_groups = newspace; + cd->named_group_list_size = newsize; } - /* For non-duplicate names, check for a duplicate number before - adding the new name. */ - - if (!dupname) - { - pcre_uchar *cslot = cd->name_table; - for (i = 0; i < cd->names_found; i++) - { - if (cslot != slot) - { - if (GET2(cslot, 0) == cd->bracount + 1) - { - *errorcodeptr = ERR65; - goto FAILED; - } - } - else i--; - cslot += cd->name_entry_size; - } - } - - PUT2(slot, 0, cd->bracount + 1); - memcpy(slot + IMM2_SIZE, name, IN_UCHARS(namelen)); - slot[IMM2_SIZE + namelen] = 0; + cd->named_groups[cd->names_found].name = name; + cd->named_groups[cd->names_found].length = namelen; + cd->named_groups[cd->names_found].number = number; + cd->names_found++; } } - /* In both pre-compile and compile, count the number of names we've - encountered. */ - - cd->names_found++; - ptr++; /* Move past > or ' */ + ptr++; /* Move past > or ' in both passes. */ goto NUMBERED_GROUP; @@ -6177,6 +7008,11 @@ for (;; ptr++) NAMED_REF_OR_RECURSE: name = ++ptr; + if (IS_DIGIT(*ptr)) + { + *errorcodeptr = ERR84; /* Group name must start with non-digit */ + goto FAILED; + } while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) ptr++; namelen = (int)(ptr - name); @@ -6189,14 +7025,14 @@ for (;; ptr++) if (lengthptr != NULL) { - const pcre_uchar *temp; + named_group *ng; if (namelen == 0) { *errorcodeptr = ERR62; goto FAILED; } - if (*ptr != terminator) + if (*ptr != (pcre_uchar)terminator) { *errorcodeptr = ERR42; goto FAILED; @@ -6207,27 +7043,29 @@ for (;; ptr++) goto FAILED; } - /* The name table does not exist in the first pass, so we cannot - do a simple search as in the code below. Instead, we have to scan the - pattern to find the number. It is important that we scan it only as - far as we have got because the syntax of named subpatterns has not - been checked for the rest of the pattern, and find_parens() assumes - correct syntax. In any case, it's a waste of resources to scan - further. We stop the scan at the current point by temporarily - adjusting the value of cd->endpattern. */ + /* The name table does not exist in the first pass; instead we must + scan the list of names encountered so far in order to get the + number. If the name is not found, set the value to 0 for a forward + reference. */ - temp = cd->end_pattern; - cd->end_pattern = ptr; - recno = find_parens(cd, name, namelen, - (options & PCRE_EXTENDED) != 0, utf); - cd->end_pattern = temp; - if (recno < 0) recno = 0; /* Forward ref; set dummy number */ + ng = cd->named_groups; + for (i = 0; i < cd->names_found; i++, ng++) + { + if (namelen == ng->length && + STRNCMP_UC_UC(name, ng->name, namelen) == 0) + break; + } + recno = (i < cd->names_found)? ng->number : 0; + + /* Count named back references. */ + + if (!is_recurse) cd->namedrefcount++; } - /* In the real compile, seek the name in the table. We check the name + /* In the real compile, search the name table. We check the name first, and then check that we have reached the end of the name in the - table. That way, if the name that is longer than any in the table, - the comparison will fail without reading beyond the table entry. */ + table. That way, if the name is longer than any in the table, the + comparison will fail without reading beyond the table entry. */ else { @@ -6240,26 +7078,78 @@ for (;; ptr++) slot += cd->name_entry_size; } - if (i < cd->names_found) /* Back reference */ + if (i < cd->names_found) { recno = GET2(slot, 0); } - else if ((recno = /* Forward back reference */ - find_parens(cd, name, namelen, - (options & PCRE_EXTENDED) != 0, utf)) <= 0) + else { *errorcodeptr = ERR15; goto FAILED; } } - /* In both phases, we can now go to the code than handles numerical - recursion or backreferences. */ + /* In both phases, for recursions, we can now go to the code than + handles numerical recursion. */ if (is_recurse) goto HANDLE_RECURSION; - else goto HANDLE_REFERENCE; + /* In the second pass we must see if the name is duplicated. If so, we + generate a different opcode. */ + if (lengthptr == NULL && cd->dupnames) + { + int count = 1; + unsigned int index = i; + pcre_uchar *cslot = slot + cd->name_entry_size; + + for (i++; i < cd->names_found; i++) + { + if (STRCMP_UC_UC(slot + IMM2_SIZE, cslot + IMM2_SIZE) != 0) break; + count++; + cslot += cd->name_entry_size; + } + + if (count > 1) + { + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; + previous = code; + *code++ = ((options & PCRE_CASELESS) != 0)? OP_DNREFI : OP_DNREF; + PUT2INC(code, 0, index); + PUT2INC(code, 0, count); + + /* Process each potentially referenced group. */ + + for (; slot < cslot; slot += cd->name_entry_size) + { + open_capitem *oc; + recno = GET2(slot, 0); + cd->backref_map |= (recno < 32)? (1 << recno) : 1; + if (recno > cd->top_backref) cd->top_backref = recno; + + /* Check to see if this back reference is recursive, that it, it + is inside the group that it references. A flag is set so that the + group can be made atomic. */ + + for (oc = cd->open_caps; oc != NULL; oc = oc->next) + { + if (oc->number == recno) + { + oc->flag = TRUE; + break; + } + } + } + + continue; /* End of back ref handling */ + } + } + + /* First pass, or a non-duplicated name. */ + + goto HANDLE_REFERENCE; + + /* ------------------------------------------------------------ */ case CHAR_R: /* Recursion */ ptr++; /* Same as (?0) */ @@ -6302,7 +7192,7 @@ for (;; ptr++) while(IS_DIGIT(*ptr)) recno = recno * 10 + *ptr++ - CHAR_0; - if (*ptr != terminator) + if (*ptr != (pcre_uchar)terminator) { *errorcodeptr = ERR29; goto FAILED; @@ -6356,8 +7246,7 @@ for (;; ptr++) if (called == NULL) { - if (find_parens(cd, NULL, recno, - (options & PCRE_EXTENDED) != 0, utf) < 0) + if (recno > cd->final_bracount) { *errorcodeptr = ERR15; goto FAILED; @@ -6406,7 +7295,7 @@ for (;; ptr++) /* Can't determine a first byte now */ - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; continue; @@ -6516,11 +7405,20 @@ for (;; ptr++) skipbytes = IMM2_SIZE; } - /* Process nested bracketed regex. Assertions used not to be repeatable, - but this was changed for Perl compatibility, so all kinds can now be - repeated. We copy code into a non-register variable (tempcode) in order to - be able to pass its address because some compilers complain otherwise. */ + /* Process nested bracketed regex. First check for parentheses nested too + deeply. */ + if ((cd->parens_depth += 1) > PARENS_NEST_LIMIT) + { + *errorcodeptr = ERR82; + goto FAILED; + } + + /* Assertions used not to be repeatable, but this was changed for Perl + compatibility, so all kinds can now be repeated. We copy code into a + non-register variable (tempcode) in order to be able to pass its address + because some compilers complain otherwise. */ + previous = code; /* For handling repetition */ *code = bravalue; tempcode = code; @@ -6540,7 +7438,9 @@ for (;; ptr++) cond_depth + ((bravalue == OP_COND)?1:0), /* Depth of condition subpatterns */ &subfirstchar, /* For possible first char */ + &subfirstcharflags, &subreqchar, /* For possible last char */ + &subreqcharflags, bcptr, /* Current branch chain */ cd, /* Tables block */ (lengthptr == NULL)? NULL : /* Actual compile phase */ @@ -6548,6 +7448,8 @@ for (;; ptr++) )) goto FAILED; + cd->parens_depth -= 1; + /* If this was an atomic group and there are no capturing groups within it, generate OP_ONCE_NC instead of OP_ONCE. */ @@ -6601,7 +7503,7 @@ for (;; ptr++) *errorcodeptr = ERR27; goto FAILED; } - if (condcount == 1) subfirstchar = subreqchar = REQ_NONE; + if (condcount == 1) subfirstcharflags = subreqcharflags = REQ_NONE; } } @@ -6650,7 +7552,9 @@ for (;; ptr++) back off. */ zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; groupsetfirstchar = FALSE; if (bravalue >= OP_ONCE) @@ -6661,28 +7565,36 @@ for (;; ptr++) no firstchar, set "none" for the whole branch. In both cases, a zero repeat forces firstchar to "none". */ - if (firstchar == REQ_UNSET) + if (firstcharflags == REQ_UNSET) { - if (subfirstchar >= 0) + if (subfirstcharflags >= 0) { firstchar = subfirstchar; + firstcharflags = subfirstcharflags; groupsetfirstchar = TRUE; } - else firstchar = REQ_NONE; - zerofirstchar = REQ_NONE; + else firstcharflags = REQ_NONE; + zerofirstcharflags = REQ_NONE; } /* If firstchar was previously set, convert the subpattern's firstchar into reqchar if there wasn't one, using the vary flag that was in existence beforehand. */ - else if (subfirstchar >= 0 && subreqchar < 0) - subreqchar = subfirstchar | tempreqvary; + else if (subfirstcharflags >= 0 && subreqcharflags < 0) + { + subreqchar = subfirstchar; + subreqcharflags = subfirstcharflags | tempreqvary; + } /* If the subpattern set a required byte (or set a first byte that isn't really the first byte - see above), set it. */ - if (subreqchar >= 0) reqchar = subreqchar; + if (subreqcharflags >= 0) + { + reqchar = subreqchar; + reqcharflags = subreqcharflags; + } } /* For a forward assertion, we take the reqchar, if set. This can be @@ -6693,7 +7605,11 @@ for (;; ptr++) of a firstchar. This is overcome by a scan at the end if there's no firstchar, looking for an asserted first char. */ - else if (bravalue == OP_ASSERT && subreqchar >= 0) reqchar = subreqchar; + else if (bravalue == OP_ASSERT && subreqcharflags >= 0) + { + reqchar = subreqchar; + reqcharflags = subreqcharflags; + } break; /* End of processing '(' */ @@ -6701,19 +7617,21 @@ for (;; ptr++) /* Handle metasequences introduced by \. For ones like \d, the ESC_ values are arranged to be the negation of the corresponding OP_values in the default case when PCRE_UCP is not set. For the back references, the values - are ESC_REF plus the reference number. Only back references and those types + are negative the reference number. Only back references and those types that consume a character may be repeated. We can test for values between ESC_b and ESC_Z for the latter; this may have to change if any new ones are ever created. */ case CHAR_BACKSLASH: tempptr = ptr; - c = check_escape(&ptr, errorcodeptr, cd->bracount, options, FALSE); + escape = check_escape(&ptr, &ec, errorcodeptr, cd->bracount, options, FALSE); if (*errorcodeptr != 0) goto FAILED; - if (c < 0) + if (escape == 0) /* The escape coded a single character */ + c = ec; + else { - if (-c == ESC_Q) /* Handle start of quoted string */ + if (escape == ESC_Q) /* Handle start of quoted string */ { if (ptr[1] == CHAR_BACKSLASH && ptr[2] == CHAR_E) ptr += 2; /* avoid empty string */ @@ -6721,71 +7639,60 @@ for (;; ptr++) continue; } - if (-c == ESC_E) continue; /* Perl ignores an orphan \E */ + if (escape == ESC_E) continue; /* Perl ignores an orphan \E */ /* For metasequences that actually match a character, we disable the setting of a first character if it hasn't already been set. */ - if (firstchar == REQ_UNSET && -c > ESC_b && -c < ESC_Z) - firstchar = REQ_NONE; + if (firstcharflags == REQ_UNSET && escape > ESC_b && escape < ESC_Z) + firstcharflags = REQ_NONE; /* Set values to reset to if this is followed by a zero repeat. */ zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; /* \g or \g'name' is a subroutine call by name and \g or \g'n' is a subroutine call by number (Oniguruma syntax). In fact, the value - -ESC_g is returned only for these cases. So we don't need to check for < - or ' if the value is -ESC_g. For the Perl syntax \g{n} the value is - -ESC_REF+n, and for the Perl syntax \g{name} the result is -ESC_k (as + ESC_g is returned only for these cases. So we don't need to check for < + or ' if the value is ESC_g. For the Perl syntax \g{n} the value is + -n, and for the Perl syntax \g{name} the result is ESC_k (as that is a synonym for a named back reference). */ - if (-c == ESC_g) + if (escape == ESC_g) { const pcre_uchar *p; + pcre_uint32 cf; + save_hwm = cd->hwm; /* Normally this is set when '(' is read */ terminator = (*(++ptr) == CHAR_LESS_THAN_SIGN)? CHAR_GREATER_THAN_SIGN : CHAR_APOSTROPHE; /* These two statements stop the compiler for warning about possibly unset variables caused by the jump to HANDLE_NUMERICAL_RECURSION. In - fact, because we actually check for a number below, the paths that + fact, because we do the check for a number below, the paths that would actually be in error are never taken. */ skipbytes = 0; reset_bracount = FALSE; - /* Test for a name */ + /* If it's not a signed or unsigned number, treat it as a name. */ - if (ptr[1] != CHAR_PLUS && ptr[1] != CHAR_MINUS) + cf = ptr[1]; + if (cf != CHAR_PLUS && cf != CHAR_MINUS && !IS_DIGIT(cf)) { - BOOL is_a_number = TRUE; - for (p = ptr + 1; *p != 0 && *p != terminator; p++) - { - if (!MAX_255(*p)) { is_a_number = FALSE; break; } - if ((cd->ctypes[*p] & ctype_digit) == 0) is_a_number = FALSE; - if ((cd->ctypes[*p] & ctype_word) == 0) break; - } - if (*p != terminator) - { - *errorcodeptr = ERR57; - break; - } - if (is_a_number) - { - ptr++; - goto HANDLE_NUMERICAL_RECURSION; - } is_recurse = TRUE; goto NAMED_REF_OR_RECURSE; } - /* Test a signed number in angle brackets or quotes. */ + /* Signed or unsigned number (cf = ptr[1]) is known to be plus or minus + or a digit. */ p = ptr + 2; while (IS_DIGIT(*p)) p++; - if (*p != terminator) + if (*p != (pcre_uchar)terminator) { *errorcodeptr = ERR57; break; @@ -6797,7 +7704,7 @@ for (;; ptr++) /* \k or \k'name' is a back reference by name (Perl syntax). We also support \k{name} (.NET syntax). */ - if (-c == ESC_k) + if (escape == ESC_k) { if ((ptr[1] != CHAR_LESS_THAN_SIGN && ptr[1] != CHAR_APOSTROPHE && ptr[1] != CHAR_LEFT_CURLY_BRACKET)) @@ -6816,13 +7723,16 @@ for (;; ptr++) not set to cope with cases like (?=(\w+))\1: which would otherwise set ':' later. */ - if (-c >= ESC_REF) + if (escape < 0) { open_capitem *oc; - recno = -c - ESC_REF; + recno = -escape; - HANDLE_REFERENCE: /* Come here from named backref handling */ - if (firstchar == REQ_UNSET) firstchar = REQ_NONE; + /* Come here from named backref handling when the reference is to a + single group (i.e. not to a duplicated name. */ + + HANDLE_REFERENCE: + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; previous = code; *code++ = ((options & PCRE_CASELESS) != 0)? OP_REFI : OP_REF; PUT2INC(code, 0, recno); @@ -6846,14 +7756,14 @@ for (;; ptr++) /* So are Unicode property matches, if supported. */ #ifdef SUPPORT_UCP - else if (-c == ESC_P || -c == ESC_p) + else if (escape == ESC_P || escape == ESC_p) { BOOL negated; - int pdata; - int ptype = get_ucp(&ptr, &negated, &pdata, errorcodeptr); - if (ptype < 0) goto FAILED; + unsigned int ptype = 0, pdata = 0; + if (!get_ucp(&ptr, &negated, &ptype, &pdata, errorcodeptr)) + goto FAILED; previous = code; - *code++ = ((-c == ESC_p) != negated)? OP_PROP : OP_NOTPROP; + *code++ = ((escape == ESC_p) != negated)? OP_PROP : OP_NOTPROP; *code++ = ptype; *code++ = pdata; } @@ -6862,7 +7772,7 @@ for (;; ptr++) /* If Unicode properties are not supported, \X, \P, and \p are not allowed. */ - else if (-c == ESC_X || -c == ESC_P || -c == ESC_p) + else if (escape == ESC_X || escape == ESC_P || escape == ESC_p) { *errorcodeptr = ERR45; goto FAILED; @@ -6873,17 +7783,18 @@ for (;; ptr++) can obtain the OP value by negating the escape value in the default situation when PCRE_UCP is not set. When it *is* set, we substitute Unicode property tests. Note that \b and \B do a one-character - lookbehind. */ + lookbehind, and \A also behaves as if it does. */ else { - if ((-c == ESC_b || -c == ESC_B) && cd->max_lookbehind == 0) + if ((escape == ESC_b || escape == ESC_B || escape == ESC_A) && + cd->max_lookbehind == 0) cd->max_lookbehind = 1; #ifdef SUPPORT_UCP - if (-c >= ESC_DU && -c <= ESC_wu) + if (escape >= ESC_DU && escape <= ESC_wu) { nestptr = ptr + 1; /* Where to resume */ - ptr = substitutes[-c - ESC_DU] - 1; /* Just before substitute */ + ptr = substitutes[escape - ESC_DU] - 1; /* Just before substitute */ } else #endif @@ -6891,8 +7802,8 @@ for (;; ptr++) so that it works in DFA mode and in lookbehinds. */ { - previous = (-c > ESC_b && -c < ESC_Z)? code : NULL; - *code++ = (!utf && c == -ESC_C)? OP_ALLANY : -c; + previous = (escape > ESC_b && escape < ESC_Z)? code : NULL; + *code++ = (!utf && escape == ESC_C)? OP_ALLANY : escape; } } continue; @@ -6902,7 +7813,7 @@ for (;; ptr++) a value > 127. We set its representation in the length/buffer, and then handle it as a data character. */ -#ifdef SUPPORT_UTF +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 if (utf && c > MAX_VALUE_FOR_SINGLE_CHAR) mclength = PRIV(ord2utf)(c, mcbuffer); else @@ -6917,8 +7828,8 @@ for (;; ptr++) /* ===================================================================*/ /* Handle a literal character. It is guaranteed not to be whitespace or # - when the extended flag is set. If we are in UTF-8 mode, it may be a - multi-byte literal character. */ + when the extended flag is set. If we are in a UTF mode, it may be a + multi-unit literal character. */ default: NORMAL_CHAR: @@ -6935,6 +7846,29 @@ for (;; ptr++) ONE_CHAR: previous = code; + + /* For caseless UTF-8 mode when UCP support is available, check whether + this character has more than one other case. If so, generate a special + OP_PROP item instead of OP_CHARI. */ + +#ifdef SUPPORT_UCP + if (utf && (options & PCRE_CASELESS) != 0) + { + GETCHAR(c, mcbuffer); + if ((c = UCD_CASESET(c)) != 0) + { + *code++ = OP_PROP; + *code++ = PT_CLIST; + *code++ = c; + if (firstcharflags == REQ_UNSET) + firstcharflags = zerofirstcharflags = REQ_NONE; + break; + } + } +#endif + + /* Caseful matches, or not one of the multicase characters. */ + *code++ = ((options & PCRE_CASELESS) != 0)? OP_CHARI : OP_CHAR; for (c = 0; c < mclength; c++) *code++ = mcbuffer[c]; @@ -6948,10 +7882,11 @@ for (;; ptr++) Otherwise, leave the firstchar value alone, and don't change it on a zero repeat. */ - if (firstchar == REQ_UNSET) + if (firstcharflags == REQ_UNSET) { - zerofirstchar = REQ_NONE; + zerofirstcharflags = REQ_NONE; zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; /* If the character is more than one byte long, we can set firstchar only if it is not to be matched caselessly. */ @@ -6959,9 +7894,16 @@ for (;; ptr++) if (mclength == 1 || req_caseopt == 0) { firstchar = mcbuffer[0] | req_caseopt; - if (mclength != 1) reqchar = code[-1] | cd->req_varyopt; + firstchar = mcbuffer[0]; + firstcharflags = req_caseopt; + + if (mclength != 1) + { + reqchar = code[-1]; + reqcharflags = cd->req_varyopt; + } } - else firstchar = reqchar = REQ_NONE; + else firstcharflags = reqcharflags = REQ_NONE; } /* firstchar was previously set; we can set reqchar only if the length is @@ -6970,9 +7912,14 @@ for (;; ptr++) else { zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; if (mclength == 1 || req_caseopt == 0) - reqchar = code[-1] | req_caseopt | cd->req_varyopt; + { + reqchar = code[-1]; + reqcharflags = req_caseopt | cd->req_varyopt; + } } break; /* End of literal character handling */ @@ -6991,7 +7938,6 @@ return FALSE; - /************************************************* * Compile sequence of alternatives * *************************************************/ @@ -7004,28 +7950,32 @@ out the amount of memory needed, as well as during the value of lengthptr distinguishes the two phases. Arguments: - options option bits, including any changes for this subpattern - codeptr -> the address of the current code pointer - ptrptr -> the address of the current pattern pointer - errorcodeptr -> pointer to error code variable - lookbehind TRUE if this is a lookbehind assertion - reset_bracount TRUE to reset the count for each branch - skipbytes skip this many bytes at start (for brackets and OP_COND) - cond_depth depth of nesting for conditional subpatterns - firstcharptr place to put the first required character, or a negative number - reqcharptr place to put the last required character, or a negative number - bcptr pointer to the chain of currently open branches - cd points to the data block with tables pointers etc. - lengthptr NULL during the real compile phase - points to length accumulator during pre-compile phase + options option bits, including any changes for this subpattern + codeptr -> the address of the current code pointer + ptrptr -> the address of the current pattern pointer + errorcodeptr -> pointer to error code variable + lookbehind TRUE if this is a lookbehind assertion + reset_bracount TRUE to reset the count for each branch + skipbytes skip this many bytes at start (for brackets and OP_COND) + cond_depth depth of nesting for conditional subpatterns + firstcharptr place to put the first required character + firstcharflagsptr place to put the first character flags, or a negative number + reqcharptr place to put the last required character + reqcharflagsptr place to put the last required character flags, or a negative number + bcptr pointer to the chain of currently open branches + cd points to the data block with tables pointers etc. + lengthptr NULL during the real compile phase + points to length accumulator during pre-compile phase -Returns: TRUE on success +Returns: TRUE on success */ static BOOL compile_regex(int options, pcre_uchar **codeptr, const pcre_uchar **ptrptr, int *errorcodeptr, BOOL lookbehind, BOOL reset_bracount, int skipbytes, - int cond_depth, pcre_int32 *firstcharptr, pcre_int32 *reqcharptr, + int cond_depth, + pcre_uint32 *firstcharptr, pcre_int32 *firstcharflagsptr, + pcre_uint32 *reqcharptr, pcre_int32 *reqcharflagsptr, branch_chain *bcptr, compile_data *cd, int *lengthptr) { const pcre_uchar *ptr = *ptrptr; @@ -7035,17 +7985,20 @@ pcre_uchar *start_bracket = code; pcre_uchar *reverse_count = NULL; open_capitem capitem; int capnumber = 0; -pcre_int32 firstchar, reqchar; -pcre_int32 branchfirstchar, branchreqchar; +pcre_uint32 firstchar, reqchar; +pcre_int32 firstcharflags, reqcharflags; +pcre_uint32 branchfirstchar, branchreqchar; +pcre_int32 branchfirstcharflags, branchreqcharflags; int length; -int orig_bracount; -int max_bracount; +unsigned int orig_bracount; +unsigned int max_bracount; branch_chain bc; bc.outer = bcptr; bc.current_branch = code; -firstchar = reqchar = REQ_UNSET; +firstchar = reqchar = 0; +firstcharflags = reqcharflags = REQ_UNSET; /* Accumulate the length for use in the pre-compile phase. Start with the length of the BRA and KET and any extra bytes that are required at the @@ -7105,8 +8058,8 @@ for (;;) into the length. */ if (!compile_branch(&options, &code, &ptr, errorcodeptr, &branchfirstchar, - &branchreqchar, &bc, cond_depth, cd, - (lengthptr == NULL)? NULL : &length)) + &branchfirstcharflags, &branchreqchar, &branchreqcharflags, &bc, + cond_depth, cd, (lengthptr == NULL)? NULL : &length)) { *ptrptr = ptr; return FALSE; @@ -7127,7 +8080,9 @@ for (;;) if (*last_branch != OP_ALT) { firstchar = branchfirstchar; + firstcharflags = branchfirstcharflags; reqchar = branchreqchar; + reqcharflags = branchreqcharflags; } /* If this is not the first branch, the first char and reqchar have to @@ -7141,23 +8096,36 @@ for (;;) we have to abandon the firstchar for the regex, but if there was previously no reqchar, it takes on the value of the old firstchar. */ - if (firstchar >= 0 && firstchar != branchfirstchar) + if (firstcharflags >= 0 && + (firstcharflags != branchfirstcharflags || firstchar != branchfirstchar)) { - if (reqchar < 0) reqchar = firstchar; - firstchar = REQ_NONE; + if (reqcharflags < 0) + { + reqchar = firstchar; + reqcharflags = firstcharflags; + } + firstcharflags = REQ_NONE; } /* If we (now or from before) have no firstchar, a firstchar from the branch becomes a reqchar if there isn't a branch reqchar. */ - if (firstchar < 0 && branchfirstchar >= 0 && branchreqchar < 0) - branchreqchar = branchfirstchar; + if (firstcharflags < 0 && branchfirstcharflags >= 0 && branchreqcharflags < 0) + { + branchreqchar = branchfirstchar; + branchreqcharflags = branchfirstcharflags; + } /* Now ensure that the reqchars match */ - if ((reqchar & ~REQ_VARY) != (branchreqchar & ~REQ_VARY)) - reqchar = REQ_NONE; - else reqchar |= branchreqchar; /* To "or" REQ_VARY */ + if (((reqcharflags & ~REQ_VARY) != (branchreqcharflags & ~REQ_VARY)) || + reqchar != branchreqchar) + reqcharflags = REQ_NONE; + else + { + reqchar = branchreqchar; + reqcharflags |= branchreqcharflags; /* To "or" REQ_VARY */ + } } /* If lookbehind, check that this branch matches a fixed-length string, and @@ -7253,7 +8221,9 @@ for (;;) *codeptr = code; *ptrptr = ptr; *firstcharptr = firstchar; + *firstcharflagsptr = firstcharflags; *reqcharptr = reqchar; + *reqcharflagsptr = reqcharflags; if (lengthptr != NULL) { if (OFLOW_MAX - *lengthptr < length) @@ -7323,19 +8293,23 @@ and the highest back reference was greater than or equ However, by keeping a bitmap of the first 31 back references, we can catch some of the more common cases more precisely. +... A second exception is when the .* appears inside an atomic group, because +this prevents the number of characters it matches from being adjusted. + Arguments: code points to start of expression (the bracket) bracket_map a bitmap of which brackets we are inside while testing; this handles up to substring 31; after that we just have to take the less precise approach - backref_map the back reference bitmap + cd points to the compile data block + atomcount atomic group level Returns: TRUE or FALSE */ static BOOL is_anchored(register const pcre_uchar *code, unsigned int bracket_map, - unsigned int backref_map) + compile_data *cd, int atomcount) { do { const pcre_uchar *scode = first_significant_code( @@ -7347,7 +8321,7 @@ do { if (op == OP_BRA || op == OP_BRAPOS || op == OP_SBRA || op == OP_SBRAPOS) { - if (!is_anchored(scode, bracket_map, backref_map)) return FALSE; + if (!is_anchored(scode, bracket_map, cd, atomcount)) return FALSE; } /* Capturing brackets */ @@ -7357,30 +8331,40 @@ do { { int n = GET2(scode, 1+LINK_SIZE); int new_map = bracket_map | ((n < 32)? (1 << n) : 1); - if (!is_anchored(scode, new_map, backref_map)) return FALSE; + if (!is_anchored(scode, new_map, cd, atomcount)) return FALSE; } - /* Other brackets */ + /* Positive forward assertions and conditions */ - else if (op == OP_ASSERT || op == OP_ONCE || op == OP_ONCE_NC || - op == OP_COND) + else if (op == OP_ASSERT || op == OP_COND) { - if (!is_anchored(scode, bracket_map, backref_map)) return FALSE; + if (!is_anchored(scode, bracket_map, cd, atomcount)) return FALSE; } + /* Atomic groups */ + + else if (op == OP_ONCE || op == OP_ONCE_NC) + { + if (!is_anchored(scode, bracket_map, cd, atomcount + 1)) + return FALSE; + } + /* .* is not anchored unless DOTALL is set (which generates OP_ALLANY) and - it isn't in brackets that are or may be referenced. */ + it isn't in brackets that are or may be referenced or inside an atomic + group. */ else if ((op == OP_TYPESTAR || op == OP_TYPEMINSTAR || op == OP_TYPEPOSSTAR)) { - if (scode[1] != OP_ALLANY || (bracket_map & backref_map) != 0) + if (scode[1] != OP_ALLANY || (bracket_map & cd->backref_map) != 0 || + atomcount > 0 || cd->had_pruneorskip) return FALSE; } /* Check for explicit anchoring */ else if (op != OP_SOD && op != OP_SOM && op != OP_CIRC) return FALSE; + code += GET(code, 1); } while (*code == OP_ALT); /* Loop for each alternative */ @@ -7398,21 +8382,24 @@ return TRUE; matching and for non-DOTALL patterns that start with .* (which must start at the beginning or after \n). As in the case of is_anchored() (see above), we have to take account of back references to capturing brackets that contain .* -because in that case we can't make the assumption. +because in that case we can't make the assumption. Also, the appearance of .* +inside atomic brackets or in a pattern that contains *PRUNE or *SKIP does not +count, because once again the assumption no longer holds. Arguments: code points to start of expression (the bracket) bracket_map a bitmap of which brackets we are inside while testing; this handles up to substring 31; after that we just have to take the less precise approach - backref_map the back reference bitmap + cd points to the compile data + atomcount atomic group level Returns: TRUE or FALSE */ static BOOL is_startline(const pcre_uchar *code, unsigned int bracket_map, - unsigned int backref_map) + compile_data *cd, int atomcount) { do { const pcre_uchar *scode = first_significant_code( @@ -7431,14 +8418,14 @@ do { switch (*scode) { case OP_CREF: - case OP_NCREF: + case OP_DNCREF: case OP_RREF: - case OP_NRREF: + case OP_DNRREF: case OP_DEF: return FALSE; default: /* Assertion */ - if (!is_startline(scode, bracket_map, backref_map)) return FALSE; + if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE; do scode += GET(scode, 1); while (*scode == OP_ALT); scode += 1 + LINK_SIZE; break; @@ -7452,7 +8439,7 @@ do { if (op == OP_BRA || op == OP_BRAPOS || op == OP_SBRA || op == OP_SBRAPOS) { - if (!is_startline(scode, bracket_map, backref_map)) return FALSE; + if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE; } /* Capturing brackets */ @@ -7462,25 +8449,40 @@ do { { int n = GET2(scode, 1+LINK_SIZE); int new_map = bracket_map | ((n < 32)? (1 << n) : 1); - if (!is_startline(scode, new_map, backref_map)) return FALSE; + if (!is_startline(scode, new_map, cd, atomcount)) return FALSE; } - /* Other brackets */ + /* Positive forward assertions */ - else if (op == OP_ASSERT || op == OP_ONCE || op == OP_ONCE_NC) + else if (op == OP_ASSERT) { - if (!is_startline(scode, bracket_map, backref_map)) return FALSE; + if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE; } - /* .* means "start at start or after \n" if it isn't in brackets that - may be referenced. */ + /* Atomic brackets */ + else if (op == OP_ONCE || op == OP_ONCE_NC) + { + if (!is_startline(scode, bracket_map, cd, atomcount + 1)) return FALSE; + } + + /* .* means "start at start or after \n" if it isn't in atomic brackets or + brackets that may be referenced, as long as the pattern does not contain + *PRUNE or *SKIP, because these break the feature. Consider, for example, + /.*?a(*PRUNE)b/ with the subject "aab", which matches "ab", i.e. not at the + start of a line. */ + else if (op == OP_TYPESTAR || op == OP_TYPEMINSTAR || op == OP_TYPEPOSSTAR) { - if (scode[1] != OP_ANY || (bracket_map & backref_map) != 0) return FALSE; + if (scode[1] != OP_ANY || (bracket_map & cd->backref_map) != 0 || + atomcount > 0 || cd->had_pruneorskip) + return FALSE; } - /* Check for explicit circumflex */ + /* Check for explicit circumflex; anything else gives a FALSE result. Note + in particular that this includes atomic brackets OP_ONCE and OP_ONCE_NC + because the number of characters matched by .* cannot be adjusted inside + them. */ else if (op != OP_CIRC && op != OP_CIRCM) return FALSE; @@ -7502,33 +8504,40 @@ return TRUE; discarded, because they can cause conflicts with actual literals that follow. However, if we end up without a first char setting for an unanchored pattern, it is worth scanning the regex to see if there is an initial asserted first -char. If all branches start with the same asserted char, or with a bracket all -of whose alternatives start with the same asserted char (recurse ad lib), then -we return that char, otherwise -1. +char. If all branches start with the same asserted char, or with a +non-conditional bracket all of whose alternatives start with the same asserted +char (recurse ad lib), then we return that char, with the flags set to zero or +REQ_CASELESS; otherwise return zero with REQ_NONE in the flags. Arguments: code points to start of expression (the bracket) + flags points to the first char flags, or to REQ_NONE inassert TRUE if in an assertion -Returns: -1 or the fixed first char +Returns: the fixed first char, or 0 with REQ_NONE in flags */ -static int -find_firstassertedchar(const pcre_uchar *code, BOOL inassert) +static pcre_uint32 +find_firstassertedchar(const pcre_uchar *code, pcre_int32 *flags, + BOOL inassert) { -register int c = -1; +register pcre_uint32 c = 0; +int cflags = REQ_NONE; + +*flags = REQ_NONE; do { - int d; + pcre_uint32 d; + int dflags; int xl = (*code == OP_CBRA || *code == OP_SCBRA || *code == OP_CBRAPOS || *code == OP_SCBRAPOS)? IMM2_SIZE:0; const pcre_uchar *scode = first_significant_code(code + 1+LINK_SIZE + xl, TRUE); - register int op = *scode; + register pcre_uchar op = *scode; switch(op) { default: - return -1; + return 0; case OP_BRA: case OP_BRAPOS: @@ -7539,10 +8548,10 @@ do { case OP_ASSERT: case OP_ONCE: case OP_ONCE_NC: - case OP_COND: - if ((d = find_firstassertedchar(scode, op == OP_ASSERT)) < 0) - return -1; - if (c < 0) c = d; else if (c != d) return -1; + d = find_firstassertedchar(scode, &dflags, op == OP_ASSERT); + if (dflags < 0) + return 0; + if (cflags < 0) { c = d; cflags = dflags; } else if (c != d || cflags != dflags) return 0; break; case OP_EXACT: @@ -7553,9 +8562,9 @@ do { case OP_PLUS: case OP_MINPLUS: case OP_POSPLUS: - if (!inassert) return -1; - if (c < 0) c = scode[1]; - else if (c != scode[1]) return -1; + if (!inassert) return 0; + if (cflags < 0) { c = scode[1]; cflags = 0; } + else if (c != scode[1]) return 0; break; case OP_EXACTI: @@ -7566,21 +8575,78 @@ do { case OP_PLUSI: case OP_MINPLUSI: case OP_POSPLUSI: - if (!inassert) return -1; - if (c < 0) c = scode[1] | REQ_CASELESS; - else if (c != scode[1]) return -1; + if (!inassert) return 0; + if (cflags < 0) { c = scode[1]; cflags = REQ_CASELESS; } + else if (c != scode[1]) return 0; break; } code += GET(code, 1); } while (*code == OP_ALT); + +*flags = cflags; return c; } /************************************************* +* Add an entry to the name/number table * +*************************************************/ + +/* This function is called between compiling passes to add an entry to the +name/number table, maintaining alphabetical order. Checking for permitted +and forbidden duplicates has already been done. + +Arguments: + cd the compile data block + name the name to add + length the length of the name + groupno the group number + +Returns: nothing +*/ + +static void +add_name(compile_data *cd, const pcre_uchar *name, int length, + unsigned int groupno) +{ +int i; +pcre_uchar *slot = cd->name_table; + +for (i = 0; i < cd->names_found; i++) + { + int crc = memcmp(name, slot+IMM2_SIZE, IN_UCHARS(length)); + if (crc == 0 && slot[IMM2_SIZE+length] != 0) + crc = -1; /* Current name is a substring */ + + /* Make space in the table and break the loop for an earlier name. For a + duplicate or later name, carry on. We do this for duplicates so that in the + simple case (when ?(| is not used) they are in order of their numbers. In all + cases they are in the order in which they appear in the pattern. */ + + if (crc < 0) + { + memmove(slot + cd->name_entry_size, slot, + IN_UCHARS((cd->names_found - i) * cd->name_entry_size)); + break; + } + + /* Continue the loop for a later or duplicate name */ + + slot += cd->name_entry_size; + } + +PUT2(slot, 0, groupno); +memcpy(slot + IMM2_SIZE, name, IN_UCHARS(length)); +slot[IMM2_SIZE + length] = 0; +cd->names_found++; +} + + + +/************************************************* * Compile a Regular Expression * *************************************************/ @@ -7602,41 +8668,55 @@ Returns: pointer to compiled data block, or NUL with errorptr and erroroffset set */ -#ifdef COMPILE_PCRE8 +#if defined COMPILE_PCRE8 PCRE_EXP_DEFN pcre * PCRE_CALL_CONVENTION pcre_compile(const char *pattern, int options, const char **errorptr, int *erroroffset, const unsigned char *tables) -#else +#elif defined COMPILE_PCRE16 PCRE_EXP_DEFN pcre16 * PCRE_CALL_CONVENTION pcre16_compile(PCRE_SPTR16 pattern, int options, const char **errorptr, int *erroroffset, const unsigned char *tables) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN pcre32 * PCRE_CALL_CONVENTION +pcre32_compile(PCRE_SPTR32 pattern, int options, const char **errorptr, + int *erroroffset, const unsigned char *tables) #endif { -#ifdef COMPILE_PCRE8 +#if defined COMPILE_PCRE8 return pcre_compile2(pattern, options, NULL, errorptr, erroroffset, tables); -#else +#elif defined COMPILE_PCRE16 return pcre16_compile2(pattern, options, NULL, errorptr, erroroffset, tables); +#elif defined COMPILE_PCRE32 +return pcre32_compile2(pattern, options, NULL, errorptr, erroroffset, tables); #endif } -#ifdef COMPILE_PCRE8 +#if defined COMPILE_PCRE8 PCRE_EXP_DEFN pcre * PCRE_CALL_CONVENTION pcre_compile2(const char *pattern, int options, int *errorcodeptr, const char **errorptr, int *erroroffset, const unsigned char *tables) -#else +#elif defined COMPILE_PCRE16 PCRE_EXP_DEFN pcre16 * PCRE_CALL_CONVENTION pcre16_compile2(PCRE_SPTR16 pattern, int options, int *errorcodeptr, const char **errorptr, int *erroroffset, const unsigned char *tables) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN pcre32 * PCRE_CALL_CONVENTION +pcre32_compile2(PCRE_SPTR32 pattern, int options, int *errorcodeptr, + const char **errorptr, int *erroroffset, const unsigned char *tables) #endif { REAL_PCRE *re; int length = 1; /* For final END opcode */ -pcre_int32 firstchar, reqchar; +pcre_int32 firstcharflags, reqcharflags; +pcre_uint32 firstchar, reqchar; +pcre_uint32 limit_match = PCRE_UINT32_MAX; +pcre_uint32 limit_recursion = PCRE_UINT32_MAX; int newline; int errorcode = 0; int skipatstart = 0; BOOL utf; +BOOL never_utf = FALSE; size_t size; pcre_uchar *code; const pcre_uchar *codestart; @@ -7653,6 +8733,11 @@ new memory is obtained from malloc(). */ pcre_uchar cworkspace[COMPILE_WORK_SIZE]; +/* This vector is used for remembering name groups during the pre-compile. In a +similar way to cworkspace, it can be expanded using malloc() if necessary. */ + +named_group named_groups[NAMED_GROUP_LIST_SIZE]; + /* Set this early so that early errors get offset 0. */ ptr = (const pcre_uchar *)pattern; @@ -7696,28 +8781,85 @@ if ((options & ~PUBLIC_COMPILE_OPTIONS) != 0) goto PCRE_EARLY_ERROR_RETURN; } +/* If PCRE_NEVER_UTF is set, remember it. */ + +if ((options & PCRE_NEVER_UTF) != 0) never_utf = TRUE; + /* Check for global one-time settings at the start of the pattern, and remember the offset for later. */ +cd->external_flags = 0; /* Initialize here for LIMIT_MATCH/RECURSION */ + while (ptr[skipatstart] == CHAR_LEFT_PARENTHESIS && ptr[skipatstart+1] == CHAR_ASTERISK) { int newnl = 0; int newbsr = 0; +/* For completeness and backward compatibility, (*UTFn) is supported in the +relevant libraries, but (*UTF) is generic and always supported. Note that +PCRE_UTF8 == PCRE_UTF16 == PCRE_UTF32. */ + #ifdef COMPILE_PCRE8 - if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF_RIGHTPAR, 5) == 0) + if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF8_RIGHTPAR, 5) == 0) { skipatstart += 7; options |= PCRE_UTF8; continue; } #endif #ifdef COMPILE_PCRE16 - if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF_RIGHTPAR, 6) == 0) + if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF16_RIGHTPAR, 6) == 0) { skipatstart += 8; options |= PCRE_UTF16; continue; } #endif +#ifdef COMPILE_PCRE32 + if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF32_RIGHTPAR, 6) == 0) + { skipatstart += 8; options |= PCRE_UTF32; continue; } +#endif + + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF_RIGHTPAR, 4) == 0) + { skipatstart += 6; options |= PCRE_UTF8; continue; } else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UCP_RIGHTPAR, 4) == 0) { skipatstart += 6; options |= PCRE_UCP; continue; } + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_NO_AUTO_POSSESS_RIGHTPAR, 16) == 0) + { skipatstart += 18; options |= PCRE_NO_AUTO_POSSESS; continue; } else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_NO_START_OPT_RIGHTPAR, 13) == 0) { skipatstart += 15; options |= PCRE_NO_START_OPTIMIZE; continue; } + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_LIMIT_MATCH_EQ, 12) == 0) + { + pcre_uint32 c = 0; + int p = skipatstart + 14; + while (isdigit(ptr[p])) + { + if (c > PCRE_UINT32_MAX / 10 - 1) break; /* Integer overflow */ + c = c*10 + ptr[p++] - CHAR_0; + } + if (ptr[p++] != CHAR_RIGHT_PARENTHESIS) break; + if (c < limit_match) + { + limit_match = c; + cd->external_flags |= PCRE_MLSET; + } + skipatstart = p; + continue; + } + + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_LIMIT_RECURSION_EQ, 16) == 0) + { + pcre_uint32 c = 0; + int p = skipatstart + 18; + while (isdigit(ptr[p])) + { + if (c > PCRE_UINT32_MAX / 10 - 1) break; /* Integer overflow check */ + c = c*10 + ptr[p++] - CHAR_0; + } + if (ptr[p++] != CHAR_RIGHT_PARENTHESIS) break; + if (c < limit_recursion) + { + limit_recursion = c; + cd->external_flags |= PCRE_RLSET; + } + skipatstart = p; + continue; + } + if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_CR_RIGHTPAR, 3) == 0) { skipatstart += 5; newnl = PCRE_NEWLINE_CR; } else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_LF_RIGHTPAR, 3) == 0) @@ -7741,8 +8883,13 @@ while (ptr[skipatstart] == CHAR_LEFT_PARENTHESIS && else break; } -/* PCRE_UTF16 has the same value as PCRE_UTF8. */ +/* PCRE_UTF(16|32) have the same value as PCRE_UTF8. */ utf = (options & PCRE_UTF8) != 0; +if (utf && never_utf) + { + errorcode = ERR78; + goto PCRE_EARLY_ERROR_RETURN2; + } /* Can't support UTF unless PCRE has been compiled to include the code. The return of an error code from PRIV(valid_utf)() is a new feature, introduced in @@ -7753,10 +8900,12 @@ not used here. */ if (utf && (options & PCRE_NO_UTF8_CHECK) == 0 && (errorcode = PRIV(valid_utf)((PCRE_PUCHAR)pattern, -1, erroroffset)) != 0) { -#ifdef COMPILE_PCRE8 +#if defined COMPILE_PCRE8 errorcode = ERR44; -#else +#elif defined COMPILE_PCRE16 errorcode = ERR74; +#elif defined COMPILE_PCRE32 + errorcode = ERR77; #endif goto PCRE_EARLY_ERROR_RETURN2; } @@ -7853,17 +9002,21 @@ cd->bracount = cd->final_bracount = 0; cd->names_found = 0; cd->name_entry_size = 0; cd->name_table = NULL; +cd->dupnames = FALSE; +cd->namedrefcount = 0; cd->start_code = cworkspace; cd->hwm = cworkspace; cd->start_workspace = cworkspace; cd->workspace_size = COMPILE_WORK_SIZE; +cd->named_groups = named_groups; +cd->named_group_list_size = NAMED_GROUP_LIST_SIZE; cd->start_pattern = (const pcre_uchar *)pattern; cd->end_pattern = (const pcre_uchar *)(pattern + STRLEN_UC((const pcre_uchar *)pattern)); cd->req_varyopt = 0; +cd->parens_depth = 0; cd->assert_depth = 0; cd->max_lookbehind = 0; cd->external_options = options; -cd->external_flags = 0; cd->open_caps = NULL; /* Now do the pre-compile. On error, errorcode will be set non-zero, so we @@ -7875,8 +9028,10 @@ outside can help speed up starting point checks. */ ptr += skipatstart; code = cworkspace; *code = OP_BRA; + (void)compile_regex(cd->external_options, &code, &ptr, &errorcode, FALSE, - FALSE, 0, 0, &firstchar, &reqchar, NULL, cd, &length); + FALSE, 0, 0, &firstchar, &firstcharflags, &reqchar, &reqcharflags, NULL, + cd, &length); if (errorcode != 0) goto PCRE_EARLY_ERROR_RETURN; DPRINTF(("end pre-compile: length=%d workspace=%d\n", length, @@ -7888,14 +9043,23 @@ if (length > MAX_PATTERN_SIZE) goto PCRE_EARLY_ERROR_RETURN; } -/* Compute the size of data block needed and get it, either from malloc or -externally provided function. Integer overflow should no longer be possible -because nowadays we limit the maximum value of cd->names_found and -cd->name_entry_size. */ +/* If there are groups with duplicate names and there are also references by +name, we must allow for the possibility of named references to duplicated +groups. These require an extra data item each. */ -size = sizeof(REAL_PCRE) + (length + cd->names_found * cd->name_entry_size) * sizeof(pcre_uchar); -re = (REAL_PCRE *)(PUBL(malloc))(size); +if (cd->dupnames && cd->namedrefcount > 0) + length += cd->namedrefcount * IMM2_SIZE * sizeof(pcre_uchar); +/* Compute the size of the data block for storing the compiled pattern. Integer +overflow should no longer be possible because nowadays we limit the maximum +value of cd->names_found and cd->name_entry_size. */ + +size = sizeof(REAL_PCRE) + + (length + cd->names_found * cd->name_entry_size) * sizeof(pcre_uchar); + +/* Get the memory. */ + +re = (REAL_PCRE *)(PUBL(malloc))(size); if (re == NULL) { errorcode = ERR21; @@ -7912,6 +9076,8 @@ re->magic_number = MAGIC_NUMBER; re->size = (int)size; re->options = cd->external_options; re->flags = cd->external_flags; +re->limit_match = limit_match; +re->limit_recursion = limit_recursion; re->first_char = 0; re->req_char = 0; re->name_table_offset = sizeof(REAL_PCRE) / sizeof(pcre_uchar); @@ -7920,6 +9086,11 @@ re->name_count = cd->names_found; re->ref_count = 0; re->tables = (tables == PRIV(default_tables))? NULL : tables; re->nullpad = NULL; +#ifdef COMPILE_PCRE32 +re->dummy = 0; +#else +re->dummy1 = re->dummy2 = re->dummy3 = 0; +#endif /* The starting points of the name/number translation table and of the code are passed around in the compile data block. The start/end pattern and initial @@ -7929,19 +9100,34 @@ field; this time it's used for remembering forward ref */ cd->final_bracount = cd->bracount; /* Save for checking forward references */ +cd->parens_depth = 0; cd->assert_depth = 0; cd->bracount = 0; cd->max_lookbehind = 0; -cd->names_found = 0; cd->name_table = (pcre_uchar *)re + re->name_table_offset; codestart = cd->name_table + re->name_entry_size * re->name_count; cd->start_code = codestart; cd->hwm = (pcre_uchar *)(cd->start_workspace); cd->req_varyopt = 0; cd->had_accept = FALSE; +cd->had_pruneorskip = FALSE; cd->check_lookbehind = FALSE; cd->open_caps = NULL; +/* If any named groups were found, create the name/number table from the list +created in the first pass. */ + +if (cd->names_found > 0) + { + int i = cd->names_found; + named_group *ng = cd->named_groups; + cd->names_found = 0; + for (; i > 0; i--, ng++) + add_name(cd, ng->name, ng->length, ng->number); + if (cd->named_group_list_size > NAMED_GROUP_LIST_SIZE) + (PUBL(free))((void *)cd->named_groups); + } + /* Set up a starting, non-extracting bracket, then compile the expression. On error, errorcode will be set non-zero, so we don't need to look at the result of the function here. */ @@ -7950,17 +9136,21 @@ ptr = (const pcre_uchar *)pattern + skipatstart; code = (pcre_uchar *)codestart; *code = OP_BRA; (void)compile_regex(re->options, &code, &ptr, &errorcode, FALSE, FALSE, 0, 0, - &firstchar, &reqchar, NULL, cd, NULL); + &firstchar, &firstcharflags, &reqchar, &reqcharflags, NULL, cd, NULL); re->top_bracket = cd->bracount; re->top_backref = cd->top_backref; re->max_lookbehind = cd->max_lookbehind; re->flags = cd->external_flags | PCRE_MODE; -if (cd->had_accept) reqchar = REQ_NONE; /* Must disable after (*ACCEPT) */ +if (cd->had_accept) + { + reqchar = 0; /* Must disable after (*ACCEPT) */ + reqcharflags = REQ_NONE; + } /* If not reached end of pattern on success, there's an excess bracket. */ -if (errorcode == 0 && *ptr != 0) errorcode = ERR22; +if (errorcode == 0 && *ptr != CHAR_NULL) errorcode = ERR22; /* Fill in the terminating state and check for disastrous overflow, but if debugging, leave the test till after things are printed out. */ @@ -7971,6 +9161,13 @@ if debugging, leave the test till after things are pri if (code - codestart > length) errorcode = ERR23; #endif +#ifdef SUPPORT_VALGRIND +/* If the estimated length exceeds the really used length, mark the extra +allocated memory as unaddressable, so that any out-of-bound reads can be +detected. */ +VALGRIND_MAKE_MEM_NOACCESS(code, (length - (code - codestart)) * sizeof(pcre_uchar)); +#endif + /* Fill in any forward references that are required. There may be repeated references; optimize for them, as searching a large regex takes time. */ @@ -7994,19 +9191,27 @@ if (cd->hwm > cd->start_workspace) } } -/* If the workspace had to be expanded, free the new memory. */ +/* If the workspace had to be expanded, free the new memory. Set the pointer to +NULL to indicate that forward references have been filled in. */ if (cd->workspace_size > COMPILE_WORK_SIZE) (PUBL(free))((void *)cd->start_workspace); +cd->start_workspace = NULL; /* Give an error if there's back reference to a non-existent capturing subpattern. */ if (errorcode == 0 && re->top_backref > re->top_bracket) errorcode = ERR15; +/* Unless disabled, check whether single character iterators can be +auto-possessified. The function overwrites the appropriate opcode values. */ + +if ((options & PCRE_NO_AUTO_POSSESS) == 0) + auto_possessify((pcre_uchar *)codestart, utf, cd); + /* If there were any lookbehind assertions that contained OP_RECURSE (recursions or subroutine calls), a flag is set for them to be checked here, -because they may contain forward references. Actual recursions can't be fixed +because they may contain forward references. Actual recursions cannot be fixed length, but subroutine calls can. It is done like this so that those without OP_RECURSE that are not fixed length get a diagnosic with a useful offset. The exceptional ones forgo this. We scan the pattern to check that they are fixed @@ -8062,33 +9267,33 @@ if (errorcode != 0) } /* If the anchored option was not passed, set the flag if we can determine that -the pattern is anchored by virtue of ^ characters or \A or anything else (such -as starting with .* when DOTALL is set). +the pattern is anchored by virtue of ^ characters or \A or anything else, such +as starting with non-atomic .* when DOTALL is set and there are no occurrences +of *PRUNE or *SKIP. Otherwise, if we know what the first byte has to be, save it, because that speeds up unanchored matches no end. If not, see if we can set the PCRE_STARTLINE flag. This is helpful for multiline matches when all branches -start with ^. and also when all branches start with .* for non-DOTALL matches. -*/ +start with ^. and also when all branches start with non-atomic .* for +non-DOTALL matches when *PRUNE and SKIP are not present. */ if ((re->options & PCRE_ANCHORED) == 0) { - if (is_anchored(codestart, 0, cd->backref_map)) - re->options |= PCRE_ANCHORED; + if (is_anchored(codestart, 0, cd, 0)) re->options |= PCRE_ANCHORED; else { - if (firstchar < 0) - firstchar = find_firstassertedchar(codestart, FALSE); - if (firstchar >= 0) /* Remove caseless flag for non-caseable chars */ + if (firstcharflags < 0) + firstchar = find_firstassertedchar(codestart, &firstcharflags, FALSE); + if (firstcharflags >= 0) /* Remove caseless flag for non-caseable chars */ { -#ifdef COMPILE_PCRE8 +#if defined COMPILE_PCRE8 re->first_char = firstchar & 0xff; -#else -#ifdef COMPILE_PCRE16 +#elif defined COMPILE_PCRE16 re->first_char = firstchar & 0xffff; +#elif defined COMPILE_PCRE32 + re->first_char = firstchar; #endif -#endif - if ((firstchar & REQ_CASELESS) != 0) + if ((firstcharflags & REQ_CASELESS) != 0) { #if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) /* We ignore non-ASCII first chars in 8 bit mode. */ @@ -8111,8 +9316,8 @@ if ((re->options & PCRE_ANCHORED) == 0) re->flags |= PCRE_FIRSTSET; } - else if (is_startline(codestart, 0, cd->backref_map)) - re->flags |= PCRE_STARTLINE; + + else if (is_startline(codestart, 0, cd, 0)) re->flags |= PCRE_STARTLINE; } } @@ -8120,17 +9325,17 @@ if ((re->options & PCRE_ANCHORED) == 0) variable length item in the regex. Remove the caseless flag for non-caseable bytes. */ -if (reqchar >= 0 && - ((re->options & PCRE_ANCHORED) == 0 || (reqchar & REQ_VARY) != 0)) +if (reqcharflags >= 0 && + ((re->options & PCRE_ANCHORED) == 0 || (reqcharflags & REQ_VARY) != 0)) { -#ifdef COMPILE_PCRE8 +#if defined COMPILE_PCRE8 re->req_char = reqchar & 0xff; -#else -#ifdef COMPILE_PCRE16 +#elif defined COMPILE_PCRE16 re->req_char = reqchar & 0xffff; +#elif defined COMPILE_PCRE32 + re->req_char = reqchar; #endif -#endif - if ((reqchar & REQ_CASELESS) != 0) + if ((reqcharflags & REQ_CASELESS) != 0) { #if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) /* We ignore non-ASCII first chars in 8 bit mode. */ @@ -8180,10 +9385,12 @@ if ((re->flags & PCRE_REQCHSET) != 0) else printf("Req char = \\x%02x%s\n", ch, caseless); } -#ifdef COMPILE_PCRE8 +#if defined COMPILE_PCRE8 pcre_printint((pcre *)re, stdout, TRUE); -#else +#elif defined COMPILE_PCRE16 pcre16_printint((pcre *)re, stdout, TRUE); +#elif defined COMPILE_PCRE32 +pcre32_printint((pcre *)re, stdout, TRUE); #endif /* This check is done here in the debugging case so that the code that @@ -8199,11 +9406,28 @@ if (code - codestart > length) } #endif /* PCRE_DEBUG */ -#ifdef COMPILE_PCRE8 +/* Check for a pattern than can match an empty string, so that this information +can be provided to applications. */ + +do + { + if (could_be_empty_branch(codestart, code, utf, cd, NULL)) + { + re->flags |= PCRE_MATCH_EMPTY; + break; + } + codestart += GET(codestart, 1); + } +while (*codestart == OP_ALT); + +#if defined COMPILE_PCRE8 return (pcre *)re; -#else +#elif defined COMPILE_PCRE16 return (pcre16 *)re; +#elif defined COMPILE_PCRE32 +return (pcre32 *)re; #endif } /* End of pcre_compile.c */ +