j*7^UddlmZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%dTd Z&dUdZ'dVdZ(e)dej*DZ+de,d<de+DZ-de,d<dWdZ.dUdZ/dUdZ0dUdZ1dUdZ2dUdZ3dUdZ4dUdZ5dUd Z6dUd!Z7dUd"Z8dUd#Z9dUd$Z:dUd%Z;dUd&Z dYdZd.Z?e d/d[d1Z@d\d3ZAd]d5ZBd^d_d9ZCd`d=ZDdad>ZEd?ejFd@fdbdDZG dcdddSZHdS)e) annotationsN) bisect_right)IncrementalDecoder) lru_cache) Generator)ENCODING_MARKSIANA_SUPPORTED_SIMILARRE_POSSIBLE_ENCODING_INDICATIONUNICODE_RANGES_COMBINED_SECONDARY_RANGE_NAMESCOMMON_CJK_CHARACTERS_LATIN_CJK_HANGUL _KATAKANA _HIRAGANA_HALFWIDTH_KATAKANA_THAI_ARABIC_ARABIC_ISOLATED_FORM _LIGATURE _SUPERSCRIPT_SENTENCE_OPEN_PUNCTUATION_ACCENT_KEYWORDS _ACCENTUATED_KNOWN_MB_DECODERS_KNOWN_MB_CLASSES _IANA_NAMES_MULTIBYTE_SEARCH_RADIUS characterstrreturnintc tj|}n#t$rYdSwxYwd}d|vr |tz}d|vr |tz}d|vr |t z}d|vr|t z}d|vr |tz}d|vr |tz}d|vr |tz}d |vr|tz}d |vr |tz}d |vs| d r |tz}d |vr |tz}|dvr |tz}t D]}||vr |t"z}n|S)zRCompute all name-based classification flags with a single unicodedata.name() call.rLATINCJKHANGULKATAKANA HALFWIDTHHIRAGANATHAIARABICz ISOLATED FORMLIGATUREz LETTER AE SUPERSCRIPT>INVERTED QUESTION MARKINVERTED EXCLAMATION MARK) unicodedataname ValueErrorrrrrrrrrrendswithrrrrr)r!descflagskws b/workspace/show-city-briefing-agent/.venv/lib/python3.11/site-packages/charset_normalizer/utils.py_character_flagsr:'s}$Y// qqE$  }}  4 T  $   ( (ET  ~~ 4  d " " * *ETT]];77   FFF ++ :: \ !E E  Ls  %%boolcJtt|tzSN)r;r:rr!s r9is_accentuatedr?Qs  ++l: ; ;;ctj|}|s|S|d}tt |ddS)N r)r2 decompositionsplitchrr$)r! decomposedcodess r9 remove_accentrIUsO!/ ::J !'',,E s58R  ! !!r@c#:K|]\}}|j|j|fVdSr=)startstop).0r3 ord_ranges r9 rOasE<<i_ind+<<<<<<r@zlist[tuple[int, int, str]]_UNICODE_RANGES_SORTEDcg|] }|d S)r)rMes r9 rTes#I#I#IQAaD#I#I#Ir@z list[int]_UNICODE_RANGE_STARTS str | Nonect|}|dkrdS|dkrdStt|dz }|dkrt|\}}}||kr|SdS)zK Retrieve the Unicode range official name from a single character. zControl characterz Basic LatinrrN)ordrrUrP)r! character_ordidxrKrLr3s r9 unicode_ranger]hsvYMr""s} ,m < s r9is_latinr_}s  ++f4 5 55r@cdtj|}d|vrdSt|}|dSd|vS)NPTF Punctuationr2categoryr]r!character_categorycharacter_ranges r9is_punctuationrhsG)29==    t"/ ":":Ou O ++r@cxtj|}d|vsd|vrdSt|}|dSd|vo|dkS)NSNTFFormsLorcres r9 is_symbolrns])29==    C+=$=$=t"/ ":":Ou o % D*<*DDr@c8t|}|dSd|vpd|vS)NF Emoticons Pictographs)r])r!rgs r9 is_emoticonrrs/"/ ":":Ou / ) M]o-MMr@cn|s|dvrdStj|}d|vp|dvS)N>｜+<>TZ>PcPdPo)isspacer2rd)r!rfs r9 is_separatorr}sOi+AAAt)29== $ $ P(:>P(PPr@cV||kSr=)islowerisupperr>s r9is_case_variablers%     )"3"3"5"5 55r@cJtt|tzSr=)r;r:rr>s r9is_cjkrs  ++d2 3 33r@cJtt|tzSr=)r;r:rr>s r9 is_hiraganar  ++i7 8 88r@cJtt|tzSr=)r;r:rr>s r9 is_katakanarrr@cJtt|tzSr=)r;r:rr>s r9 is_hangulr  ++g5 6 66r@cJtt|tzSr=)r;r:rr>s r9is_thairs  ++e3 4 44r@cJtt|tzSr=)r;r:rr>s r9 is_arabicrrr@cJtt|tzSr=)r;r:rr>s r9is_arabic_isolated_formrs  ++.CC D DDr@c|tvSr=)rr>s r9is_cjk_uncommonrs 1 11r@ range_namec|tvSr=)r )rs r9is_unicode_range_secondaryrs / //r@cn| o | o |dko|dkS)Nu)r| isprintabler>s r9is_unprintablersL      "%%'' ' "   "  ! r@ sequencebytes | bytearray search_zonect|ttfstt |}|dt ||}|}d|vrd|vrdS|dd}tj |D]W}| d dd }tj |}||cSXdS) zW Extract using ASCII-only decoder any specified encoding in the first n-bytes. Nscodingscharsetasciiignoreerrorsr-_) isinstancebytes bytearray TypeErrorlenminlowerdecoder finditergroupreplacerget) rrseq_len search_bytes lowered_bytes decoded_zonematchspecified_encoding encoding_ianas r9any_specified_encodingrs h 2 3 3x==G7c';7778L &&((M %%*M*I*It$++GH+EEL09,GG!!"[[^^1133;;CEE#(:;;  $  % 4r@)maxsizer3c|tvrdStD]J} tj||n#t t tf$rYDwxYwdSdS)zQ Verify is a specific encoding is a multi byte one based on it IANA name TF)rr importlib import_modulegetcodec ImportErrorAttributeError LookupError)r3providers r9is_multi_byte_encodingrs  !!!t&   #H - - 6 6t < < < <^[9    H tt 5s'>AAtuple[str | None, bytes]ctD]I}t|}t|tr|g}|D]}||r||fccS JdS)z9 Identify and extract SIG/BOM in given sequence. )Nr@)r rr startswith)r iana_encodingmarksmarks r9identify_sig_or_bomrs (++ %3M%B eU # # GE + +D""4(( +$d****** + + 9r@rc |dvS)N>utf_16utf_32rR)rs r9should_strip_sig_or_bomr(s  4 44r@Tcp_namestrictc|dd}tj|}||S|rt d|d|S)zIReturns the Python normalized encoding name (Not the IANA official name).rrNzUnable to retrieve IANA for '')rrrrr4)rrrs r9 iana_namer,sbmmoo%%c3//GOG,,M  ECCCCDDD Nr@ iana_name_a iana_name_bfloatct|st|rdStjd|j}tjd|j}|d}|d}d}t dD]C}t |g}||||kr|dz }D|dz S)Ngz encodings.rrrr)rrrrrangerr) rr decoder_a decoder_bid_aid_bcharacter_match_counti to_be_decodeds r9 cp_similarityr:sk**.D[.Q.Qs'(B[(B(BCCVI'(B[(B(BCCVI(y999D(y999D!" 3ZZ''$aSzz ;;} % %])C)C C C !Q & ! 3 &&r@c2|tvo|t|vS)z Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. )r )rrs r9 is_cp_similarrNs% -- ? 1+> >r@charset_normalizerz)%(asctime)s | %(levelname)s | %(message)slevel format_stringNonectj|}||tj}|tj|||dSr=)logging getLoggersetLevel StreamHandler setFormatter Formatter addHandler)r3rrloggerhandlers r9set_logging_handlerrYsm  t $ $F OOE#%%G *=99::: gr@F sequencesroffsetsr chunk_sizebom_or_sig_availablestrip_sig_or_bom sig_payloadris_multi_byte_decoderdecoded_payloaddeferred_decodingGenerator[str, None, None]c #K|rX|drCt|} t|} |D] } | | z| z} || | |z}|sdS|V!dS|r|s|D]} || | |z}|sdS|VdS| rF|s|n|t|d}|D]&} || | |z}|sdSt||V'dS|D]@} | |z}|t|dzkr|| | |z}|r|s||z}|||rdnd}|r| dkrt |d}|d|}d}|rot|} | | zt|z}t d|t z }t | |t z}||||dk}|rU|sS||vrOt| | d z d D]:}|||}|r|s||z}||d}|d||vrn;|VBdS) Niso2022_rrrrrCF) rrr"rrmaxr findr)rrrrrrrrrrdecoded_lengthsequence_lengthrdecoded_offsetchunk base_bytes cut_sequence chunk_endchunk_partial_size_chk chunk_prefix found_nearbyexpected_offset search_start search_endjs r9cut_sequence_chunksrfsnQ=33J??Q_--i..  A/?BN#N^j5P$PQE KKKK    H!6H  A#AJ$67E KKKK    B. PII9S=M=M=O=O3P  3 3A%a!j.&89L lM22 2 2 2 2  3 3 2 2 AJI3y>>A---$QZ%78L# :,< :*\9  ''#8Fxxh(E% "Q.1*b.A.A&$%<&<%<= $ " %(%9%9N&'.&8C NN&JO#&q/rs"""""""%%%%%%8''''T<<<<""""6