a allapex Regular Expressions Alex Nuijten

Transcription

1 a allapex Regular Expressions say what? Alex Nuijten

2 a nuijten.blogspot.com

3 Matt Brown

4 create table t (name varchar2(25)) / insert into t (name) values ('BLAKE'); insert into t (name) values ('BLOKE'); insert into t (name) values ('BLEAKE'); insert into t (name) values ('BLKE'); insert into t (name) values ('JAKESBLAKE'); insert into t (name) values ('BLAKESJAKE'); insert into t (name) values ('BL_KE'); insert into t (name) values ('BL%KE');

5 select name from t where name = 'BLAKE' / NAME BLAKE

6 select name from t where name = 'BL_KE' / NAME BL_KE

7 select name from t where name = 'BL%KE' / NAME BL%KE

8 select name from t where name like 'BLAKE' / NAME BLAKE

9 select name from t where name like 'BL_KE' / NAME BLAKE BLOKE BL_KE BL%KE

10 select name from t where name like 'BL%KE' / NAME BLAKE BLOKE BLEAKE BLKE BL_KE BL%KE BLAKESJAKE

11 select name from t where name like 'BL\_KE' ESCAPE '\' / NAME BL_KE

12 select name from t where name like 'BL"_KE' ESCAPE '"' / NAME BL_KE

13 REGEXP_

14 REGEXP_LIKE REGEXP_SUBSTR REGEXP_INSTR REGEXP_REPLACE REGEXP_COUNT

15 Joe Penniston

16 create table theme_parks (id number,name varchar2(50),description clob );

17 Aquatica Orlando Epcot Islands of Adventure LEGOLAND Florida Disney's Magic Kingdom Discovery Cove Universal Studios Florida Disney's Animal Kingdom SeaWorld Orlando Disney's Hollywood Studios

18 At the Animal Kingdom, Disney elevates the concept of a zoological park to something completely new. Here you can take a safari ride through the wilds of Africa -- complete with real animals, raft down the Amazon or ride a runaway train through Mount Everest. The park is home to about 1,500 animals representing 250 species Osceola Parkway Lake Buena Vista, FL Region: Disney Area Phone: Toll-Free:

19 At the Animal Kingdom, Disney elevates the concept of a zoological park to something completely new. Here you can take a safari ride through the wilds of Africa -- complete with real animals, raft down the Amazon or ride a runaway train through Mount Everest. The park is home to about 1,500 animals representing 250 species Osceola Parkway Lake Buena Vista, FL Region: Disney Area Phone: Toll-Free:

20 Metacharacters.^$*+?

21 . Any Character * Zero or More Occurrences [] Characters in List + One or More Occurrences ^ Beginning of String? Zero or One Occurrence $ End of String {m} Matches m Times XOR

22 Phone: Toll-Free:

23 select id,name,regexp_substr (description, ' ') phone from theme_parks; ID NAME PHONE Aquatica Orlando Epcot 3 Islands of Adventure : LEGOLAND Florida Disney's Magic Kingdom 6 Discovery Cove Universal Studios Florida Disney's Animal Kingdom SeaWorld Orlando Disney's Hollywood Studios rows selected.

24 6000 Universal Blvd. Orlando, FL Region: Orlando Phone: (407) Toll-Free:

25 6000 Universal Blvd. Orlando, FL Region: Orlando Phone: (407) Toll-Free: [

26 6000 Universal Blvd. Orlando, FL Region: Orlando Phone: (407) Toll-Free: [

27 [Bracket Expressions] {Interval Qualifiers}

28 [Specify List of Characters]

29 [ ]

30 [ ]

31 [0-9]

32 [A-Z]

33 [A-Za-z]

34 [0-9][0-9][0-9]-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]

35 {m}

36 {m,}

37 {m,n}

38 Repeat exactly three times Everything in this List [0-9]{3}-[0-9]{3}-[0-9]{4} Hyphen

39 select id,name,regexp_substr (description,'[0-9]{3}-[0-9]{3}-[0-9]{4}') phone from theme_parks; ID NAME PHONE Aquatica Orlando Epcot 3 Islands of Adventure LEGOLAND Florida Disney's Magic Kingdom 6 Discovery Cove Universal Studios Florida Disney's Animal Kingdom SeaWorld Orlando Disney's Hollywood Studios rows selected.

40 200 Epcot Center Drive Lake Buena Vista, FL Region: Disney Area Phone: Toll-Free: [0-9]{3}-[0-9]{3}-[0-9]{4} Hyphen

41 Alter ations

42 or Hyphen Space (- )

43 200 Epcot Center Drive Lake Buena Vista, FL Region: Disney Area Phone: Toll-Free: [0-9]{3}(- )[0-9]{3}(- )[0-9]{4}

44 select id,name,regexp_substr (description,'[0-9]{3}(- )[0-9]{3}(- )[0-9]{4}') phone from theme_parks; ID NAME PHONE Aquatica Orlando Epcot Islands of Adventure LEGOLAND Florida Disney's Magic Kingdom 6 Discovery Cove Universal Studios Florida Disney's Animal Kingdom SeaWorld Orlando Disney's Hollywood Studios

45 Lake Buena Vista, FL Region: Disney Area Phone: Toll-Free:

46 or Hyphen Space (-.) or Any Character

47 or Hyphen Space (- \.) or Period

48 select id,name,regexp_substr (description, '[0-9]{3}(- \.)[0-9]{3}(- \.)[0-9]{4}') phone from theme_parks; ID NAME PHONE Aquatica Orlando Epcot Islands of Adventure LEGOLAND Florida Disney's Magic Kingdom Discovery Cove Universal Studios Florida Disney's Animal Kingdom SeaWorld Orlando Disney's Hollywood Studios rows selected.

49 [:Character Classes:]

50 [0-9]{3} [[:digit:]]{3} Only in Bracket Expressions

51 [:alnum:] [:alpha:] [:upper:] [:lower:] [:digit:] [:space:] [:print:] [:punct:] [:graph:] [:cntrl:] [:xdigit:]

52 select id,name,regexp_substr (description,'\(?[[:digit:]]{3}\)?(- \.)([[:digit:]]{3} [[:alpha:]]{3})(- \.),regexp_substr (description,'\(?([[:digit:]]{3} [[:digit:]]{1}[[:alpha:]]{2})\)?(- \.)([[:digi from theme_parks;

53 ID NAME PHONE PHONE Aquatica Orlando Epcot Islands of Adventure (407) LEGOLAND Florida LEGO 5 Disney's Magic Kingdom Discovery Cove DI-SCO-VERY 7 Universal Studios Florida Disney's Animal Kingdom SeaWorld Orlando Disney's Hollywood Studios rows selected.

54 First Word

55 LEGOLAND Florida, the largest LEGOLAND park in the world, features 50 rides, shows and attractions spread out over the LEGO-strewn 150-acres. If you're traveling with younger kids, LEGOLAND makes a great choice since its designed for kids between the ages of 2 and 12. Be sure to stop by the Big Shop to bring some of the LEGO action home. One LEGOLAND Way Winter Haven, FL Region: Polk County Phone: Toll-Free: LEGO

56 One or More Any Character Space.+[[:space:]]

57 select regexp_substr (description, '.+[[:space:]]') first_word from theme_parks where id = 4; FIRST_WORD LEGOLAND Florida, the largest LEGOLAND park in the world...

58 Any Character Except a Space Sweetie187

59 Negates Bracket Expression [^[:space:]]

60 select regexp_substr (description, '[^[:space:]]*') first_word from theme_parks where id = 4; FIRST_WORD LEGOLAND

61 Split Strings Phil Thomas

62 create table test (str varchar2(500)) / insert into test values ('ABC,DEF,GHI,JKL,MNO') / commit /

63 REGEXP_LIKE REGEXP_SUBSTR REGEXP_INSTR REGEXP_REPLACE REGEXP_COUNT

64 REGEXP_LIKE REGEXP_SUBSTR REGEXP_INSTR REGEXP_REPLACE REGEXP_COUNT

65 [A-Z]+ [A-Za-z]+ [A-Za-z0-9]+ [[:alnum:]]+ [^,]+

66 select regexp_substr (str, '[^,]+') split from test / SPLIT ABC 1 row selected.

67 Starting from select regexp_substr (str, '[^,]+', 1, 2) split from test / SPLIT DEF Second Occurence 1 row selected.

68 select regexp_substr (str, '[^,]+', 1, rownum) split from test connect by level <= regexp_count (str, ',') + 1 / SPLIT ABC DEF GHI JKL MNO How many comma's in the String? 5 rows selected.

69 insert into test values ('123,456,789') / commit /

70 select regexp_substr (str, '[^,]+', 1, rownum) split from test connect by level <= regexp_count (str, ',') + 1 / SPLIT ABC DEF GHI JKL MNO

71 select regexp_substr (str, '[^,]+', 1, rn) split from test cross join (select rownum rn from test connect by level <= regexp_count (str, ',') + 1 ) where regexp_substr (str, '[^,]+', 1, rn) is not null / SPLIT ABC 123 DEF 456 GHI 789 JKL MNO 8 rows selected.

72 select split from test,lateral (select regexp_substr (str, '[^,]+', 1, rownum) split from dual connect by level <= regexp_count(str, ',') + 1); SPLIT ABC DEF GHI JKL MNO rows selected.

73 James MacDonald Pattern Matching

74 SQL> create table weather 2 (dt date not null 3,rain number not null 4 ); Table created.

75 SQL> insert into weather 2 with rsf(r) 3 as 4 (select 1 5 from dual 6 union all 7 select r from rsf 9 where r < 31) 10 select to_date (to_char (r) ' ', 'dd-mm-yyyy') dt 11,round ((dbms_random.value * 20)) rain 12 from rsf 13 / 31 rows created.

76 SQL> select dt 2,rain 3,trend 4 from weather 5 match_recognize( 6 order by dt 7 measures classifier() as trend 8 all rows per match 9 pattern (Better* Worse* same*) 10 define Better as Better.rain < prev (rain) 11,Worse as Worse.rain > prev (rain) 12,same as same.rain = prev (rain) 13 );

77 DT RAIN TREND AUG AUG-14 0 BETTER 03-AUG WORSE 04-AUG-14 6 BETTER 05-AUG WORSE 06-AUG-14 1 BETTER 07-AUG WORSE 08-AUG SAME 09-AUG AUG AUG AUG AUG AUG AUG BETTER 18 WORSE 9 BETTER 4 BETTER 17 WORSE 16 BETTER 5 BETTER What about Analytic Functions?

78 SQL> select dt 2,rain 3,case 4 when rain < lag (rain) over (order by dt) 5 then 'Better' 6 when rain > lag (rain) over (order by dt) 7 then 'Worse' 8 when rain = lag (rain) over (order by dt) 9 then 'Same' 10 end trend 11 from weather;

79 DT RAIN TREND AUG AUG-14 0 Better 03-AUG Worse 04-AUG-14 6 Better 05-AUG Worse 06-AUG-14 1 Better 07-AUG Worse 08-AUG Same 09-AUG Better 10-AUG Worse 11-AUG AUG AUG AUG AUG-14 9 Better 4 Better 17 Worse 16 Better 5 Better and what does this have to do with Regular Expressions?

80 Looking for Dry Spells SQL> select * 2 from weather 3 match_recognize 4 ( 5 order by dt 6 measures first (wet.dt) as first_wetday 7,dry.dt as dryday 8,last (wet.dt) as last_wetday 9 one row per match 10 pattern (wet dry{2,} wet*) 11 define 12 wet as wet.rain > 10, 13 dry as dry.rain <= );

81 pattern (wet dry{2,} wet*) FIRST_WET DRYDAY LAST_WETD AUG AUG AUG AUG AUG AUG-14 2 rows selected. 09-AUG AUG AUG AUG AUG AUG AUG AUG AUG AUG AUG AUG AUG AUG AUG-14 7

82 Mutant669 Data Redaction

83 select from Redacted Data

84 SQL> alter table emp 2 add varchar2 (35) 3 / Table altered. SQL> update emp 2 set = lower(ename) '.' lower (ename) '@company.com' 3 / 14 rows updated. SQL> commit 2 / Commit complete.

85 SQL> select ename 2, 3 from emp 4 / ENAME SMITH smith.smith@company.com ALLEN allen.allen@company.com WARD ward.ward@company.com JONES jones.jones@company.com MARTIN martin.martin@company.com BLAKE blake.blake@company.com CLARK clark.clark@company.com SCOTT scott.scott@company.com KING king.king@company.com TURNER turner.turner@company.com ADAMS adams.adams@company.com JAMES james.james@company.com FORD ford.ford@company.com MILLER miller.miller@company.com 14 rows selected.

86

87 begin dbms_redact.add_policy (object_schema => 'ALEX',object_name => 'EMP',policy_name => 'Hide Address',column_name => ' ',function_type => dbms_redact.regexp,expression => '1=1',regexp_pattern => => => dbms_redact.re_beginning,regexp_occurrence => dbms_redact.re_all,enable => true ); end;

88 SQL> select ename 2, 3 from alex.emp 4 / ENAME SMITH yy@company.com ALLEN yy@company.com WARD yy@company.com JONES yy@company.com MARTIN yy@company.com BLAKE yy@company.com CLARK yy@company.com SCOTT yy@company.com KING yy@company.com TURNER yy@company.com ADAMS yy@company.com JAMES yy@company.com FORD yy@company.com MILLER yy@company.com 14 rows selected.

89 -- Preset values for "regexp_pattern" parameter for use in -- regular expression redaction (function_type := dbms_redact.regexp) -- In general, this value determines what to recognize and replace. RE_PATTERN_US_SSN CONSTANT VARCHAR2(26) := '(\d\d\d)-(\d\d)-(\d\d\d\d)'; RE_PATTERN_CC_L6_T4 CONSTANT VARCHAR2(33) := '(\d\d\d\d\d\d)(\d\d\d*)(\d\d\d\d)'; RE_PATTERN_ANY_DIGIT CONSTANT VARCHAR2(2) := '\d'; RE_PATTERN_US_PHONE CONSTANT VARCHAR2(39) := '(\(\d\d\d\) \d\d\d)-(\d\d\d)-(\d\d\d\d)'; RE_PATTERN_ _ADDRESS CONSTANT VARCHAR2(51) := RE_PATTERN_IP_ADDRESS CONSTANT VARCHAR2(36) := '(\d{1,3}\.\d{1,3}\.\d{1,3})\.\d{1,3}';

90 -- Preset values for "regexp_pattern" parameter for use in -- regular expression redaction (function_type := dbms_redact.regexp) -- In general, this value determines what to recognize and replace. RE_PATTERN_US_SSN CONSTANT VARCHAR2(26) := '(\d\d\d)-(\d\d)-(\d\d\d\d)'; RE_PATTERN_CC_L6_T4 CONSTANT VARCHAR2(33) := '(\d\d\d\d\d\d)(\d\d\d*)(\d\d\d\d)'; RE_PATTERN_ANY_DIGIT CONSTANT VARCHAR2(2) := '\d'; RE_PATTERN_US_PHONE C O N S T A N TV A R C H A R 2 ( 3 9 ) : = '(\(\d\d\d\) \d\d\d)-(\d\d\d)-(\d\d\d\d)'; RE_PATTERN_ _ADDRESS CONSTANT VARCHAR2(51) := '([A-Za-z0-9._%+-]+)@([A-Za-z0-9.-]+\.[A-Za-z]{2,4})'; RE_PATTERN_IP_ADDRESS CONSTANT VARCHAR2(36) := '(\d{1,3}\.\d{1,3}\.\d{1,3})\.\d{1,3}';

91 begin dbms_redact.add_policy (object_schema,object_name,policy_name,column_name,function_type,expression,regexp_pattern => 'ALEX' => 'EMP' => 'Hide Address' => ' ' => dbms_redact.regexp => '1=1' => dbms_redact.re_pattern_ _address,regexp_replace_string => dbms_redact.re_redact_ _name,regexp_position => dbms_redact.re_beginning,regexp_occurrence => dbms_redact.re_all,enable => true ); end;

92 SQL> select ename 2, 3 from emp 4 / ENAME SMITH xxxx@company.com ALLEN xxxx@company.com WARD xxxx@company.com JONES xxxx@company.com MARTIN xxxx@company.com BLAKE xxxx@company.com CLARK xxxx@company.com SCOTT xxxx@company.com KING xxxx@company.com TURNER xxxx@company.com ADAMS xxxx@company.com JAMES xxxx@company.com FORD xxxx@company.com MILLER xxxx@company.com 14 rows selected.

93 SQL> select regexp_pattern 2,regexp_replace_string 3 from redaction_columns 4 where object_name = 'EMP' 5 and object_owner = user 6 / REGEXP_PATTERN REGEXP_REPLACE_STRING ([A-Za-z0-9._%+-]+)@([A-Za-z0-9.-]+\.[A-Za-z]{2,4}) xxxx@\2

94 Some other built-in Patterns

95 SQL> select ename 2,hiredate 3,credit_card 4 from emp 5 / ENAME HIREDATE CREDIT_CARD SMITH 17-DEC ALLEN 20-FEB WARD 22-FEB JONES 02-APR MARTIN 28-SEP BLAKE 01-MAY CLARK 09-JUN SCOTT 19-APR

96 begin dbms_redact.add_policy (object_schema => 'ALEX',object_name => 'EMP',policy_name => 'Hide Creditcard',expression => '1=1',column_name => 'CREDIT_CARD',function_type => dbms_redact.regexp,regexp_pattern => dbms_redact.re_pattern_any_digit,regexp_replace_string => 'X' ); end;

97 ENAME HIREDATE CREDIT_CARD SMITH 17-DEC-80 XXXXXXXXXXXXXXXX ALLEN 20-FEB-81 XXXXXXXXXXXXXXXX WARD 22-FEB-81 XXXXXXXXXXXXXXXX JONES 02-APR-81 XXXXXXXXXXXXXXXX MARTIN 28-SEP-81 XXXXXXXXXXXXXXXX BLAKE 01-MAY-81 XXXXXXXXXXXXXXXX CLARK 09-JUN-81 XXXXXXXXXXXXXXX SCOTT 19-APR-87 XXXXXXXXXXXXXXX

98 begin dbms_redact.add_policy (object_schema => 'ALEX',object_name => 'EMP',policy_name => 'Hide Creditcard',expression => '1=1',column_name => 'CREDIT_CARD',function_type => dbms_redact.partial,function_parameters => dbms_redact.redact_ccn16_f12 ); end;

99 ENAME HIREDATE CREDIT_CARD SMITH 17-DEC-80 ****-****-****-2 ALLEN 20-FEB-81 ****-****-****-9 WARD 22-FEB-81 ****-****-****-8 JONES 02-APR-81 ****-****-****-7 MARTIN 28-SEP-81 ****-****-****-0 BLAKE 01-MAY-81 ****-****-****-7 CLARK 09-JUN-81 ****-****-****- SCOTT 19-APR-87 ****-****-****-

100 begin dbms_redact.add_policy( object_schema,object_name,column_name,policy_name,function_type,function_parameters,expression,regexp_pattern => 'ALEX' => 'EMP' => 'CREDIT_CARD' => 'Hide Credit Card' => dbms_redact.regexp => null => '1=1' => dbms_redact.re_pattern_cc_l6_t4,regexp_replace_string => dbms_redact.re_redact_cc_middle_digits,regexp_position => dbms_redact.re_beginning,regexp_occurrence => dbms_redact.re_first ); end; /

101 ENAME HIREDATE CREDIT_CARD SMITH 17-DEC XXXXXX4172 ALLEN 20-FEB XXXXXX1949 WARD 22-FEB XXXXXX0198 JONES 02-APR XXXXXX8587 MARTIN 28-SEP XXXXXX5920 BLAKE 01-MAY XXXXXX9737 CLARK 09-JUN XXXXXX5275 SCOTT 19-APR XXXXXX7948

102 begin dbms_redact.add_policy( object_schema => 'ALEX',object_name => 'EMP',column_name => 'CREDIT_CARD',policy_name => 'gibberish',function_type => DBMS_REDACT.RANDOM,expression => '1=1' ); END; /

103 ENAME HIREDATE CREDIT_CARD SMITH 17-DEC-80 _Z^\1LYqgbfd9Tay ALLEN 20-FEB-81 dn9vpp_.lva^xxg6 WARD JONES 02-APR-81 \~+bn/mad'3jv7ui MARTIN 28-SEP-81 BLAKE 01-MAY-81 *PJ",qL2XR*(4yT CLARK 09-JUN-81!e5`E)g))m=hM1o SCOTT 19-APR-87 * V]>n%y2%^eFv-

104 ENAME HIREDATE CREDIT_CARD SMITH 17-DEC-80 ALLEN 20-FEB-81 WARD 22-FEB-81 {?-Z#/b0$G5b+7j8 JONES 02-APR-81 IKn4cRczQSv<kZk# MARTIN 28-SEP-81 qx=>90;?y`efa8cg BLAKE 01-MAY-81 otsnzmvhw\[v#u^p CLARK 09-JUN-81 <$/9d5T'D,7Al=\ SCOTT 19-APR-87 au$o]"p i},we*x

105 a word of warning

106 select from Redacted Data

107 SQL> select ename 2,hiredate 3,credit_card 4 from emp 5 where substr (credit_card, 1, 1) = 4 6 /

108 ENAME HIREDATE CREDIT_CARD SMITH 17-DEC-80 XXXXXXXXXXXXXXXX ALLEN 20-FEB-81 XXXXXXXXXXXXXXXX WARD 22-FEB-81 XXXXXXXXXXXXXXXX

109

110 and finally

111

112

113 regexplained.co.uk

114

115

116

117

118 regexp_replace

119 Thank You

120 a allapex? Alex Nuijten nuijten.blogspot.com