
    j                       U d dl mZ d dlmZ d dlmZ ddlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddlmZmZmZm Z m!Z!m"Z"m#Z# eez  ez  ez  ez  Z$de%d<    G d	 d
          Z& ed          d7d            Z'd  e(d          D             Z)de%d<    G d d          Z* G d de*          Z+ G d de*          Z, G d de*          Z- G d de*          Z. G d d e*          Z/ G d! d"e*          Z0 G d# d$e*          Z1 G d% d&e*          Z2 G d' d(e*          Z3 G d) d*e*          Z4 ed          d8d/            Z5	 d9d:d6Z6dS );    )annotations)	lru_cache)	getLogger   )COMMON_CJK_CHARACTERSCOMMON_SAFE_ASCII_CHARACTERSTRACECompatibleFamillyRange_ACCENTUATED_ARABIC_ARABIC_ISOLATED_FORM&_BASIC_LATIN_COMPATIBLE_RANGE_FAMILIES_CJK_COMPATIBLE_RANGE_FAMILIES#_COMPATIBLE_WITH_ANY_RANGE_FAMILIES_HANGUL_HALFWIDTH_KATAKANA	_HIRAGANA	_KATAKANA_LATIN	_LIGATURE_RANGE_FAMILIES_SENTENCE_OPEN_PUNCTUATION_SUPERSCRIPT_THAI)_character_flagsis_emoticonis_punctuationis_separator	is_symbolremove_accentunicode_rangeint_GLYPH_MASKc                  8   e Zd ZU dZdZded<   ded<   ded<   ded<   ded	<   ded
<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded <   ded!<   ded"<   d'd%Zd&S )(CharInfoz>Pre-computed character properties shared across all detectors.)	character	printablealphaupperlowerspacedigitis_asciicase_variableflagsaccentuatedlatinis_cjkis_katakanais_halfwidth_katakana	is_arabicis_ligatureis_superscriptis_sentence_open_punctuationis_glyphpunctsymrangesepemoticonsafe
common_cjk
unaccentedstrr'   boolr(   r)   r*   r+   r,   r-   r.   r/   r#   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   
str | Noner=   r>   r?   r@   rA   rB   returnNonec                
   || _         t          |          }|dk     rd| _        d| _        || _        d| _        d| _        |t          v | _        d| _	        d| _
        d| _        d| _        d| _        d| _        d| _        d| _        d|cxk    rdk    rWn nTd| _        d| _        d| _        d| _        d| _        d| _        d| _        t0          | _        d| _        d| _        d| _        n@d|cxk    rdk    rWn nTd| _        d| _        d| _        d| _        d| _        d| _        d| _        t0          | _        d| _        d| _        d| _        nd|cxk    rd	k    rRn nOd| _        d| _        d| _        d| _        d| _        d| _        d| _        d
| _        d| _        d| _        d| _        n}|dk    sd|cxk    rdk    rVn nSd| _        d| _        d| _        d| _        d| _        |dk    | _        d| _        d
| _        d| _        d| _        d| _        n|                                | _        d| _        d| _        d| _        d| _        d| _        d| _        d
| _        d| _        | j        rt=          |          nd| _        | j        rt?          |          nd| _        nd| _        d| _        |                                | _        |                                 | _        |!                                | _        |"                                | _        |#                                | _        |$                                | _        | j        | j        k    | _        tK          |          }| j        rd| _        ntM          |          | _        || _        tO          |tP          z            | _        tO          |t0          z            | _        tO          |tR          z            | _	        tO          |tT          z            | _
        tO          |tV          z            | _        tO          |tX          z            | _        tO          |tZ          z            | _        tO          |t\          z            | _        tO          |t^          z            | _        tO          |t`          z            | _        | j        r| j        rtc          |          | _        n|| _        | j	        o|td          v | _        | j        r)t=          |          | _        t?          |          | _        nd| _        d| _        tg          |          | _4        tk          |          | _6        dS )zkCompute all properties for *character* (built once per codepoint,
        every branch assigns every slot).   TFA   Z   a   z   0   9   r       	      N)7r'   ordr.   r1   rB   r?   rA   r   r@   r3   r4   r5   r6   r7   r8   r9   r:   r)   r*   r+   r,   r-   r(   r/   r   r0   r2   r;   r<   isprintabler   r    isalphaisupperislowerisspaceisdigitr   r   rD   r   r   r   r   r   r   r   r   r$   r!   r   r"   r=   r   r>   )selfr'   or0   s       _/workspace/show-city-briefing-agent/.venv/lib/python3.11/site-packages/charset_normalizer/md.py__init__zCharInfo.__init__k   s    # Ys77 DM$D'DO!DM#DO!%AADIDK$D).D&"DN$D"'D05D-!DMQ}}}}"}}}}}!
!
"
"
"
!%%)"#
!
"
 qC!
"
!
"
"
!%%)"#
!
"
 qB"
"
"
"
!
!%%*"
"
"
 bQ!\\\\r\\\\\"
"
"
!
"
!"b%*"
"
"
  "+!6!6!8!8"
"
"
"
"
%*"
"
:>.S^I666e
37>L9Y///u "DMDI&2244DN"**,,DJ"**,,DJ"**,,DJ"**,,DJ"**,,DJ!%tz!9D *)44Ez 7 % +I 6 6DJ#EL$899Defn--DJut|,,DK#EI$566D)-e6I.I)J)JD&!%'/22DN#EI$566D"&u|';"<"<D04U=W5W0X0XD- !455DMz ,d. ,"/	":":"+"kPi;P.PDO ~ !+I66
$Y//"
 "9--
	**    N)r'   rC   rF   rG   )__name__
__module____qualname____doc__	__slots____annotations__r]    r^   r\   r&   r&   ,   sV        HHI> NNNOOOKKKKKKKKKKKKKKKNNNJJJKKKLLLOOO&&&&NNNKKKIIIIIINNNJJJOOOH+ H+ H+ H+ H+ H+r^   r&   N)maxsizer'   rC   rF   c                     t          |           S )zBBuild (once per codepoint) and cache the CharInfo for *character*.)r&   )r'   s    r\   
_char_inforh      s     Ir^   c                F    g | ]}t          t          |                    S re   )r&   chr).0
_codepoints     r\   
<listcomp>rm      s3     $ $ $",HS__$ $ $r^   rI   zlist[CharInfo]_ASCII_CHAR_INFOc                  >    e Zd ZdZdZdd	Zdd
Zedd            ZdS )MessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    re   r'   rC   infor&   rF   rG   c                    t           )z
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        NotImplementedErrorrZ   r'   rq   s      r\   	feed_infozMessDetectorPlugin.feed_info  s
    
 "!r^   c                    t           )zB
        Permit to reset the plugin to the initial state.
        rs   rZ   s    r\   resetzMessDetectorPlugin.reset  s
     "!r^   floatc                    t           )z
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        rs   rx   s    r\   ratiozMessDetectorPlugin.ratio  s
     "!r^   Nr'   rC   rq   r&   rF   rG   rF   rG   rF   rz   )	r_   r`   ra   rb   rc   rv   ry   propertyr|   re   r^   r\   rp   rp     sm         
 I" " " "" " " " " " " X" " "r^   rp   c                  B    e Zd ZdZddZdd	Zdd
Zedd            ZdS ) TooManySymbolOrPunctuationPlugin_punctuation_count_symbol_count_character_count_last_printable_charrF   rG   c                >    d| _         d| _        d| _        d | _        d S Nr   r   rx   s    r\   r]   z)TooManySymbolOrPunctuationPlugin.__init__*  s(    '("#%&04!!!r^   r'   rC   rq   r&   c                    | xj         dz  c_         || j        k    rD|j        s=|j        r| xj        dz  c_        n%|j        s|j        r|j        s| xj        dz  c_        || _        dS )1Optimized feed using pre-computed character info.r      N)	r   r   r@   r;   r   r-   r<   r?   r   ru   s      r\   rv   z*TooManySymbolOrPunctuationPlugin.feed_info1  s    "111$)1z (''1,'''Z (DH (T] (""a'""$-!!!r^   c                0    d| _         d| _        d| _        d S r   )r   r   r   rx   s    r\   ry   z&TooManySymbolOrPunctuationPlugin.reset=  s     "# !r^   rz   c                ^    | j         dk    rdS | j        | j        z   | j         z  }|dk    r|ndS )Nr           333333?)r   r   r   )rZ   ratio_of_punctuations     r\   r|   z&TooManySymbolOrPunctuationPlugin.ratioB  sK     A%%3 #d&88!'" (<s'B'B##Kr^   Nr~   r}   r   	r_   r`   ra   rc   r]   rv   ry   r   r|   re   r^   r\   r   r   "  s}        I5 5 5 5
. 
. 
. 
.   
 L L L XL L Lr^   r   c                  B    e Zd ZdZddZdd	Zdd
Zedd            ZdS )TooManyAccentuatedPluginr   _accentuated_countrF   rG   c                "    d| _         d| _        d S r   r   rx   s    r\   r]   z!TooManyAccentuatedPlugin.__init__Q  s    %&'(r^   r'   rC   rq   r&   c                X    | xj         dz  c_         |j        r| xj        dz  c_        dS dS r   r   N)r   r1   r   ru   s      r\   rv   z"TooManyAccentuatedPlugin.feed_infoU  sE    " 	)##q(####	) 	)r^   c                "    d| _         d| _        d S r   r   rx   s    r\   ry   zTooManyAccentuatedPlugin.reset\  s     !"#r^   rz   c                N    | j         dk     rdS | j        | j         z  }|dk    r|ndS )N   r   gffffff?r   )rZ   ratio_of_accentuations     r\   r|   zTooManyAccentuatedPlugin.ratio`  s<     1$$3'+'>AV'V(=(E(E$$3Nr^   Nr~   r}   r   r   re   r^   r\   r   r   N  s{        :I) ) ) )) ) ) )$ $ $ $ O O O XO O Or^   r   c                  B    e Zd ZdZddZdd	Zdd
Zedd            ZdS )UnprintablePlugin_unprintable_countr   _has_escaperF   rG   c                0    d| _         d| _        d| _        d S Nr   Fr   rx   s    r\   r]   zUnprintablePlugin.__init__l  s     '(%&!&r^   r'   rC   rq   r&   c                    |dk    rd| _         |j        s#|j        s|dk    r|dk    r| xj        dz  c_        | xj        dz  c_        dS )r   Tu   ﻿r   N)r   r(   r,   r   r   ru   s      r\   rv   zUnprintablePlugin.feed_infoq  sq    #D 	)J	) V##X%%##q(##"r^   c                "    d| _         d| _        d S r   )r   r   rx   s    r\   ry   zUnprintablePlugin.reset  s    "# r^   rz   c                R    | j         dk    rdS | j        rdS | j        dz  | j         z  S )Nr   r         ?r   )r   r   r   rx   s    r\   r|   zUnprintablePlugin.ratio  s;     A%%3 	3'!+t/DDDr^   Nr~   r}   r   r   re   r^   r\   r   r   i  s{        II' ' ' '
# # # #! ! ! ! E E E XE E Er^   r   c                  B    e Zd ZdZddZdd	Zdd
Zedd            ZdS )SuspiciousDuplicateAccentPlugin_successive_countr   _last_latin_character_last_was_accentuatedrF   rG   c                >    d| _         d| _        d | _        d| _        d S r   r   rx   s    r\   r]   z(SuspiciousDuplicateAccentPlugin.__init__  s(    &'%&6:"+0"""r^   r'   rC   rq   r&   c                   | xj         dz  c_         | j        V|j        rO| j        rH|j        r| j        j        r| xj        dz  c_        |j        | j        j        k    r| xj        dz  c_        || _        |j        | _        dS r   )r   r   r1   r   r*   r   rB   ru   s      r\   rv   z)SuspiciousDuplicateAccentPlugin.feed_info  s    "&2  3* 3 z ,d8> ,&&!+&&$"<"GGG&&!+&&%)"%)%5"""r^   c                >    d| _         d| _        d | _        d| _        d S r   r   rx   s    r\   ry   z%SuspiciousDuplicateAccentPlugin.reset  s(    !" !%)"%*"""r^   rz   c                @    | j         dk    rdS | j        dz  | j         z  S )Nr   r   r   )r   r   rx   s    r\   r|   z%SuspiciousDuplicateAccentPlugin.ratio  s+     A%%3&*d.CCCr^   Nr~   r}   r   r   re   r^   r\   r   r     s}        I1 1 1 16 6 6 6+ + + + D D D XD D Dr^   r   c                  B    e Zd ZdZddZdd	Zdd
Zedd            ZdS )SuspiciousRange"_suspicious_successive_range_countr   _last_printable_seen_last_printable_rangerF   rG   c                >    d| _         d| _        d | _        d | _        d S r   r   rx   s    r\   r]   zSuspiciousRange.__init__  s(    78/%&04!15"""r^   r'   rC   rq   r&   c                0   | xj         dz  c_         |j        s|j        s|j        rd| _        d| _        dS | j        || _        |j        | _        dS | j        }|j        }||k    s| t          ||          r| xj        dz  c_        || _        || _        dS r   )	r   r,   r;   r@   r   r   r=    is_suspiciously_successive_ranger   )rZ   r'   rq   unicode_range_aunicode_range_bs        r\   rv   zSuspiciousRange.feed_info  s    ": 	 	ty 	(,D%)-D&F$,(1D%)-D&F&*&@&*j o--1H/QQ =771<77$-!%4"""r^   c                >    d| _         d| _        d | _        d | _        d S r   )r   r   r   r   rx   s    r\   ry   zSuspiciousRange.reset  s(     !23/$(!%)"""r^   rz   c                D    | j         dk    rdS | j        dz  | j         z  }|S )NrR   r   r   )r   r   )rZ   ratio_of_suspicious_range_usages     r\   r|   zSuspiciousRange.ratio  s8     B&&3 3a7!2"' /.r^   Nr~   r}   r   r   re   r^   r\   r   r     sw        I6 6 6 65 5 5 52* * * * / / / X/ / /r^   r   c                  B    e Zd ZdZddZdd	Zdd
Zedd            ZdS )SuperWeirdWordPlugin_word_count_bad_word_count_foreign_long_count_is_current_word_bad_foreign_long_watchr   _bad_character_count_buffer_length_buffer_last_char_buffer_last_char_accentuated_buffer_accent_count_buffer_glyph_count_buffer_upper_count_buffer_first_lower_buffer_has_non_ascii_buffer_last_char_ligature_buffer_has_internal_ligature_is_current_word_invalid_invalid_word_countrF   rG   c                   d| _         d| _        d| _        d| _        d| _        d| _        d| _        d| _        d | _        d| _	        d| _
        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d S r   r   rx   s    r\   r]   zSuperWeirdWordPlugin.__init__
  s     !$%() */!). %&)*!#$-138*)*!() () ). +0"05'38*.3%()   r^   r'   rC   rq   r&   c                
   |j         r| j        rd| _        |j        | _        | j        dk    r|j        | _        | xj        dz  c_        || _        |j        r| xj	        dz  c_	        |j
        sd| _        |j        | _        |j        r| xj        dz  c_        |j        r| xj        dz  c_        n| j        s|j        r|j        rd| _        dS | j        sdS |j        s|j        r| j        rd| _        d| _        |j        s|j        s|j        r| xj        dz  c_        | j        }| xj        |z  c_        |dk    r| j        |z  dk    rd| _        n| j        r<| j                                        r#| j	        |k    r| xj        dz  c_        d| _        nV| j        dk    rd| _        | xj        dz  c_        n3| j        r,| j        r%| j	        |dz
  k    r| xj        dz  c_        d| _        |dk    r9| j        r2| j	        dk    o| j	        |z  dk    }|s| xj        dz  c_        d| _        | j        r'| xj        dz  c_        | xj        |z  c_        d	| _        | j        r| xj         dz  c_         d	| _        d	| _        d| _        d| _        d	| _        d| _        d| _        d| _	        d	| _        d	| _        d	| _        d	| _        dS |d
vr5|j!        s0|j"        r+d| _        | xj        dz  c_        || _        d	| _        dS dS dS dS )r   Tr   r   N         ?   r   F>   _-<=>|~)#r)   r   r   r7   r   r+   r   r   r*   r   r.   r   r1   r   r   r:   r   r   r2   r9   r8   r   r   r,   r;   r>   r   r   rV   r   r   r   r   r-   r<   )rZ   r'   rq   buffer_lengthprobable_camel_caseds        r\   rv   zSuperWeirdWordPlugin.feed_info"  s   : 	. :592.2.>D+"a''+/:(1$%.D"z .((A-((= 2-1*151AD. /))Q.))} 0((A-(((- 0tz 0TEU 0+/(F" 	F, 	1	1$($F	1 )-D%,0D): B	7 B	7tx B	7!!%!4M!!]2!!!!,}<CC04D--65.66885 0MAA,,1,,04D---2204D-,,1,,,.	50	5 0MA4EEE
 ,,1,,04D-""t'?",q0 H0=@CG %
 , 5,,1,,04D-( 2$$)$$))]:)),1), 6((A-((05-',D$"#D%)D"16D.()D%'(D$'(D$',D$).D&.3D+16D...@@@J A A )-D%1$%.D"16D... A@@@@@r^   c                   d| _         d | _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d| _	        d| _
        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d S r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rx   s    r\   ry   zSuperWeirdWordPlugin.reset  s    !%-2*$)!#(   !$%!#$ $%!#$ #$ #( %*"*/'-2*(-%#$   r^   rz   c                b    | j         rdS | j        dk    r| j        dk    rdS | j        | j        z  S )Nr   
   r   r   )r   r   r   r   r   rx   s    r\   r|   zSuperWeirdWordPlugin.ratio  sC    # 	3r!!d&>!&C&C3(4+@@@r^   Nr~   r}   r   r   re   r^   r\   r   r     s        I,* * * *0b7 b7 b7 b7H% % % %* A A A XA A Ar^   r   c                  F    e Zd ZdZdZddZdd
ZddZedd            Z	dS )CjkUncommonPluginz<
    Detect messy CJK text that probably means nothing.
    r   _uncommon_countrF   rG   c                "    d| _         d| _        d S r   r   rx   s    r\   r]   zCjkUncommonPlugin.__init__  s    %&$%r^   r'   rC   rq   r&   c                X    | xj         dz  c_         |j        s| xj        dz  c_        dS dS r   )r   rA   r   ru   s      r\   rv   zCjkUncommonPlugin.feed_info  sD    " 	&  A%    	& 	&r^   c                "    d| _         d| _        d S r   r   rx   s    r\   ry   zCjkUncommonPlugin.reset  s     ! r^   rz   c                T    | j         dk     rdS | j        | j         z  }|dk    r|dz  ndS )Nr   r   r      r   )rZ   uncommon_form_usages     r\   r|   zCjkUncommonPlugin.ratio  sC     1$$3%)%9D<Q%Q +>*C*C"Q&&Lr^   Nr~   r}   r   
r_   r`   ra   rb   rc   r]   rv   ry   r   r|   re   r^   r\   r   r     s          8I& & & && & & &! ! ! ! M M M XM M Mr^   r   c                  F    e Zd ZdZdZddZdd
ZddZedd            Z	dS )SuspiciousKatakanaPluginzDDetect implausible halfwidth Katakana and uncommon CJK combinations._katakana_count_halfwidth_katakana_count
_cjk_count_uncommon_cjk_countrF   rG   c                >    d| _         d| _        d| _        d| _        d S r   r   rx   s    r\   r]   z!SuspiciousKatakanaPlugin.__init__  s'    $%./& ()   r^   r'   rC   rq   r&   c                    |j         r)| xj        dz  c_        |j        r| xj        dz  c_        dS | xj        dz  c_        |j        s| xj        dz  c_        dS dS r   )r4   r   r5   r   r   rA   r   ru   s      r\   rv   z"SuspiciousKatakanaPlugin.feed_info  s     	  A%  ) 4..!3..F1 	*$$)$$$$	* 	*r^   c                >    d| _         d| _        d| _        d| _        d S r   r   rx   s    r\   ry   zSuspiciousKatakanaPlugin.reset  s'     )*&#$   r^   rz   c                t    | j         dk    r,| j         | j        k    rd| j        cxk    r| j        k    rn ndS dS )Nr      r   r   )r   r   r   r   rx   s    r\   r|   zSuspiciousKatakanaPlugin.ratio  sT     *a//.$2FFFT_@@@@(@@@@@@3sr^   Nr~   r}   r   r   re   r^   r\   r   r     s}        NNI* * * *
* 
* 
* 
*% % % %    X  r^   r   c                  B    e Zd ZdZddZdd	Zdd
Zedd            ZdS )ArchaicUpperLowerPlugin_buf_character_count_since_last_sep_successive_upper_lower_count#_successive_upper_lower_count_finalr   _last_alpha_seen_upper_last_alpha_seen_lower_current_ascii_onlyrF   rG   c                v    d| _         d| _        d| _        d| _        d| _        d| _        d| _        d| _        d S )NFr   Tr  rx   s    r\   r]   z ArchaicUpperLowerPlugin.__init__  sG    	45,23*890%&,1#,1#)-   r^   r'   rC   rq   r&   c                >   |j         o|j        }| }|rg| j        dk    r\| j        dk    r#|j        s| j        s| xj        | j        z  c_        d| _        d| _        d| _        | xj        dz  c_        d| _        dS | j        r|j	        sd| _        | j        dk    rJ|j
        r| j        s|j        r.| j        r'| j        r| xj        dz  c_        d| _        nd| _        nd| _        | xj        dz  c_        | xj        dz  c_        |j
        | _        |j        | _        dS )r   r   @   Fr   TNr   )r)   r/   r  r-   r	  r  r  r  r   r.   r*   r  r+   r  )rZ   r'   rq   is_concerned	chunk_seps        r\   rv   z!ArchaicUpperLowerPlugin.feed_info  su   !Z>D,>**	 	=AA4::
 ;0 ; 88688 23D.34D0DI!!Q&!!'+D$F# 	-DM 	-',D$/!33
 	"t: 	"
	"#:	" 9 %66!;66 %DII $DII!	",,1,,&*j#&*j###r^   c                v    d| _         d| _        d| _        d| _        d| _        d| _        d| _        d| _        d S )Nr   FT)r   r  r  r  r  r  r  r	  rx   s    r\   ry   zArchaicUpperLowerPlugin.reset;  sG     !/0,-.*340&+#&+#	#'   r^   rz   c                :    | j         dk    rdS | j        | j         z  S )Nr   r   )r   r  rx   s    r\   r|   zArchaicUpperLowerPlugin.ratioE  s&     A%%37$:OOOr^   Nr~   r}   r   r   re   r^   r\   r  r    s~        	I. . . .)1 )1 )1 )1V( ( ( ( P P P XP P Pr^   r  c                  B    e Zd ZdZddZddZdd
Zedd            ZdS )ArabicIsolatedFormPluginr   _isolated_form_countrF   rG   c                "    d| _         d| _        d S r   r  rx   s    r\   r]   z!ArabicIsolatedFormPlugin.__init__P  s    %&)*!!!r^   c                "    d| _         d| _        d S r   r  rx   s    r\   ry   zArabicIsolatedFormPlugin.resetT  s     !$%!!!r^   r'   rC   rq   r&   c                h    | xj         dz  c_         |j        t          z  r| xj        dz  c_        dS dS r   )r   r0   r   r  ru   s      r\   rv   z"ArabicIsolatedFormPlugin.feed_infoX  sJ    ":-- 	+%%*%%%%	+ 	+r^   rz   c                >    | j         dk     rdS | j        | j         z  }|S )Nr   r   r  )rZ   isolated_form_usages     r\   r|   zArabicIsolatedFormPlugin.ratio_  s,     1$$3%)%>AV%V""r^   Nr~   r}   r   )	r_   r`   ra   rc   r]   ry   rv   r   r|   re   r^   r\   r  r  M  su        <I+ + + +& & & &+ + + + # # # X# # #r^   r  r   rE   r   rD   c                    | |dS t           |          }t           |         }||k    rdS |t          v s	|t          v rdS t          ||          t          v rdS | dk    r	|t          vS |dk    r	|t          vS dS )z_
    Determine if two Unicode ranges seen next to each other can be considered suspicious.
    NTFzBasic Latin)r   r   r
   r   r   )r   r   	familly_a	familly_bs       r\   r   r   i  s     /"9t$_5I$_5IIu 	888;;;ui337QQQu -'' FFF-'' FFF4r^   皙?Fdecoded_sequencemaximum_thresholdrz   debugc           
        t          |           }|dk     rd}n|dk     rd}nd}|                                 }t          }t          }t	                      }t                      }	t                      }
t                      }t                      }t                      }t                      }t                      }t                      }t                      }|j        }|	j        }|
j        }|j        }|j        }|j        }|j        }|j        }|j        }|j        }t          d||          D ]X}| |||z            D ]}t!          |          }|dk     r	||         }n ||          } |||            |||           |r|j        r |||           Y |||           |j        r |||            |||           |j        re |||           |j        r |||           |j        r |||            |||           n|j        r |||           |j        r |||           |j        |	j        z   |
j        z   |j        z   |j        z   |j        z   |j        z   |j        z   |j        z   |j        z   } | |k    r nZ|d         }! |d|!           |s |d|!            |d|!           |j        |	j        z   |
j        z   |j        z   |j        z   |j        z   |j        z   |j        z   |j        z   |j        z   } |rt1          d	          }"|"                    t4          d
| d|  d|            |dk    rL|"                    t4          d| dd                     |"                    t4          d| dd                     ||	|
|||||||f
D ],}#|"                    t4          |#j         d|#j                    -t9          | d          S )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    i  rP   i   r  rI   r   r   
charset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=   zStarting with: NzEnding with: iz: r   )lenisasciirn   rh   r   r   r   r   r   r   r   r   r  r  rv   r=   rS   r(   r)   r2   r3   r4   r6   r|   r   logr	   	__class__round)$r  r  r   seq_lenstepis_pure_ascii
ascii_info	char_infod_spd_tad_upd_sdad_srd_swd_cud_skd_aud_ai	d_sp_feed	d_ta_feed	d_up_feed
d_sda_feed	d_sr_feed	d_sw_feed	d_cu_feed	d_sk_feed	d_au_feed	d_ai_feedblock_startr'   	codepointrq   mean_mess_rationl_infologgerdts$                                       r\   
mess_ratiorI    s    '((G}}	4
 +2244M
 "JI .N-O-OD%=%?%?D/11D-L-N-NE+--D!5!7!7D/11D%=%?%?D$;$=$=D%=%?%?D IIIJIIIIIIQ.. N
 N
)+d8J*JK )	/ )	/I !^^I3!), y++ Ii&&&Ii&&& > /Ii...Ii&&& ~ +	)T***	)T*** z /	)T***: 0Jy$///; /Ii...Ii....% /Ii...> /Ii... Jjj k j	
 j j j j j	 	 ///E 0 R.	$    	%IdG$$$	$    Jjj k j	
 j j j j j	 	  =/00

5155 5HW5 5!25 5	
 	
 	
 R<<JJuG0@"0EGGHHHJJuG.>suu.EGGHHHtUD$dD$O 	= 	=BJJu;;;;<<<<!$$$r^   )r'   rC   rF   r&   )r   rE   r   rE   rF   rD   )r  F)r  rC   r  rz   r   rD   rF   rz   )7
__future__r   	functoolsr   loggingr   constantr   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   utilsr   r   r   r   r    r!   r"   r$   rd   r&   rh   r=   rn   rp   r   r   r   r   r   r   r   r   r  r  r   rI  re   r^   r\   <module>rO     s   " " " " " " "                                                         .                  '>I-	9EA A A A AG+ G+ G+ G+ G+ G+ G+ G+X 4   $ $05c

$ $ $     
" " " " " " " "<)L )L )L )L )L'9 )L )L )LXO O O O O1 O O O6"E "E "E "E "E* "E "E "EJ)D )D )D )D )D&8 )D )D )DX6/ 6/ 6/ 6/ 6/( 6/ 6/ 6/rpA pA pA pA pA- pA pA pAfM M M M M* M M MD+ + + + +1 + + +\TP TP TP TP TP0 TP TP TPn# # # # #1 # # #8 4       H JOZ% Z% Z% Z% Z% Z% Z%r^   