U
    iv                     @  s  U d dl mZ d dlZd dlmZ d dlmZ ejdkrFd dlm	Z	 n.zd dl
m	Z	 W n ek
rr   dd Z	Y nX d	d
lmZmZmZmZmZmZmZmZmZmZmZmZmZ d	dlmZmZmZmZmZm Z m!Z! eeB eB eB eB Z"de#d< e	G dd dZ$G dd dZ%e	G dd de%Z&e	G dd de%Z'e	G dd de%Z(e	G dd de%Z)e	G dd de%Z*e	G dd de%Z+e	G dd de%Z,e	G d d! d!e%Z-e	G d"d# d#e%Z.ed$d%d&d&d'd(d)d*Z/ed+d%d3d.d/d'd/d0d1d2Z0dS )4    )annotationsN)	lru_cache)	getLogger)      )finalc                 C  s   | S )N )clsr   r   9/tmp/pip-unpacked-wheel-mwc9rwws/charset_normalizer/md.pyr      s    r      )COMMON_CJK_CHARACTERSCOMMON_SAFE_ASCII_CHARACTERSTRACEUNICODE_SECONDARY_RANGE_KEYWORD_ACCENTUATED_ARABIC_ARABIC_ISOLATED_FORM_CJK_HANGUL	_HIRAGANA	_KATAKANA_LATIN_THAI)_character_flagsis_emoticonis_punctuationis_separator	is_symbolremove_accentunicode_rangeint_GLYPH_MASKc                   @  s2   e Zd ZdZdZddddZdddd	d
ZdS )CharInfou{  Pre-computed character properties shared across all detectors.

    Instantiated once and reused via :meth:`update` on every character
    in the hot loop so that redundant calls to str methods
    (``isalpha``, ``isupper``, …) and cached utility functions
    (``_character_flags``, ``is_punctuation``, …) are avoided when
    several plugins need the same information.
    	character	printablealphaupperlowerspacedigitis_asciicase_variableflagsaccentuatedlatinis_cjk	is_arabicis_glyphpunctsymNonereturnc                 C  sj   d| _ d| _d| _d| _d| _d| _d| _d| _d| _d| _	d| _
d| _d| _d| _d| _d| _d| _d S )N Fr   r#   selfr   r   r
   __init__N   s"    zCharInfo.__init__str)r$   r7   c                 C  s
  || _ t|}|dk r(d| _d| _d| _d| _d| _d|  krJdkrn nFd| _d| _d| _	d| _
d| _d| _d| _t| _d| _d| _d| _qd|  krdkrn nFd| _d| _d| _	d| _
d| _d| _d| _t| _d| _d| _d| _qd|  kr
d	krRn nDd| _d| _d| _	d| _
d| _d| _d| _d
| _d| _d| _d| _n|dksxd|  krtdkrn nHd| _d| _d| _	d| _
d| _|dk| _d| _d
| _d| _d| _d| _nf| | _d| _d| _d| _	d| _
d| _d| _d
| _d| _| jr
t|nd| _| jr t|nd| _nd| _| | _| | _| | _| | _	| | _
| | _| j	| jk| _| jrt|}nd
}|| _t|t@ | _t|t@ | _t|t@ | _t|t@ | _t|t @ | _| jrt|nd| _| jr t|nd| _dS )zBUpdate all properties for *character* (called once per character).   TFA   Z   a   z   0   9   r       	      N)!r$   ordr+   r.   r0   r1   r2   r&   r'   r(   r)   r*   r%   r,   r   r-   r/   r3   r4   isprintabler   r   isalphaisupperislowerisspaceisdigitr   boolr   r   r   r!   )r:   r$   or-   r   r   r
   updatea   s    


&








zCharInfo.updateN)__name__
__module____qualname____doc__	__slots__r;   rP   r   r   r   r
   r"   /   s   	r"   c                   @  sF   e Zd ZdZdZddddddZdd	d
dZedd	ddZdS )MessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    r   r<   r"   r5   r$   infor7   c                 C  s   t dS )z
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        NNotImplementedErrorr:   r$   rX   r   r   r
   	feed_info   s    zMessDetectorPlugin.feed_infor6   c                 C  s   t dS )zB
        Permit to reset the plugin to the initial state.
        NrY   r9   r   r   r
   reset   s    zMessDetectorPlugin.resetfloatc                 C  s   t dS )z
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        NrY   r9   r   r   r
   ratio   s    zMessDetectorPlugin.ratioN)	rQ   rR   rS   rT   rU   r\   r]   propertyr_   r   r   r   r
   rV      s   rV   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS ) TooManySymbolOrPunctuationPlugin_punctuation_count_symbol_count_character_count_last_printable_charZ_frenzy_symbol_in_wordr5   r6   c                 C  s"   d| _ d| _d| _d | _d| _d S Nr   Frb   r9   r   r   r
   r;      s
    z)TooManySymbolOrPunctuationPlugin.__init__r<   r"   rW   c                 C  sb   |  j d7  _ || jkrX|tkrX|jr6|  jd7  _n"|jsX|jrXt|sX|  jd7  _|| _dS )1Optimized feed using pre-computed character info.r      N)	re   rf   r   r3   rc   r*   r4   r   rd   r[   r   r   r
   r\      s    z*TooManySymbolOrPunctuationPlugin.feed_infoc                 C  s   d| _ d| _d| _d S Nr   )rc   re   rd   r9   r   r   r
   r]     s    z&TooManySymbolOrPunctuationPlugin.resetr^   c                 C  s0   | j dkrdS | j| j | j  }|dkr,|S dS )Nr           333333?)re   rc   rd   )r:   Zratio_of_punctuationr   r   r
   r_     s    

z&TooManySymbolOrPunctuationPlugin.ratioN	rQ   rR   rS   rU   r;   r\   r]   r`   r_   r   r   r   r
   ra      s   ra   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )TooManyAccentuatedPluginre   _accentuated_countr5   r6   c                 C  s   d| _ d| _d S rj   ro   r9   r   r   r
   r;   "  s    z!TooManyAccentuatedPlugin.__init__r<   r"   rW   c                 C  s&   |  j d7  _ |jr"|  jd7  _dS rh   r   N)re   r.   rp   r[   r   r   r
   r\   &  s    z"TooManyAccentuatedPlugin.feed_infoc                 C  s   d| _ d| _d S rj   ro   r9   r   r   r
   r]   -  s    zTooManyAccentuatedPlugin.resetr^   c                 C  s*   | j dk rdS | j| j  }|dkr&|S dS )Nr   rk   gffffff?ro   )r:   Zratio_of_accentuationr   r   r
   r_   1  s    
zTooManyAccentuatedPlugin.ratioNrm   r   r   r   r
   rn     s   rn   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )UnprintablePlugin_unprintable_countre   r5   r6   c                 C  s   d| _ d| _d S rj   rs   r9   r   r   r
   r;   >  s    zUnprintablePlugin.__init__r<   r"   rW   c                 C  s<   |j s*|js*|dkr*|dkr*|  jd7  _|  jd7  _dS )rh   u   ﻿r   N)r)   r%   rt   re   r[   r   r   r
   r\   B  s    zUnprintablePlugin.feed_infoc                 C  s
   d| _ d S rj   )rt   r9   r   r   r
   r]   M  s    zUnprintablePlugin.resetr^   c                 C  s   | j dkrdS | jd | j  S )Nr   rk   r   )re   rt   r9   r   r   r
   r_   P  s    
zUnprintablePlugin.ratioNrm   r   r   r   r
   rr   :  s   rr   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )SuspiciousDuplicateAccentPlugin_successive_countre   _last_latin_character_last_was_accentuatedr5   r6   c                 C  s   d| _ d| _d | _d| _d S rg   rw   r9   r   r   r
   r;   a  s    z(SuspiciousDuplicateAccentPlugin.__init__r<   r"   rW   c                 C  st   |  j d7  _ | jdk	rb|jrb| jrb|jrB| j rB|  jd7  _t|t| jkrb|  jd7  _|| _|j| _dS rq   )re   ry   r.   rz   r'   rJ   rx   r   r[   r   r   r
   r\   h  s    z)SuspiciousDuplicateAccentPlugin.feed_infoc                 C  s   d| _ d| _d | _d| _d S rg   rw   r9   r   r   r
   r]   w  s    z%SuspiciousDuplicateAccentPlugin.resetr^   c                 C  s   | j dkrdS | jd | j  S )Nr   rk   ri   )re   rx   r9   r   r   r
   r_   }  s    
z%SuspiciousDuplicateAccentPlugin.ratioNrm   r   r   r   r
   rv   X  s   rv   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )SuspiciousRange"_suspicious_successive_range_countre   _last_printable_seen_last_printable_ranger5   r6   c                 C  s   d| _ d| _d | _d | _d S rj   r|   r9   r   r   r
   r;     s    zSuspiciousRange.__init__r<   r"   rW   c                 C  s   |  j d7  _ |js"|js"|tkr2d| _d| _dS | jdkrP|| _t|| _dS | j}t|}t||rv|  jd7  _|| _|| _dS rq   )	re   r)   r3   r   r~   r   r    is_suspiciously_successive_ranger}   )r:   r$   rX   unicode_range_aunicode_range_br   r   r
   r\     s    


zSuspiciousRange.feed_infoc                 C  s   d| _ d| _d | _d | _d S rj   )re   r}   r~   r   r9   r   r   r
   r]     s    zSuspiciousRange.resetr^   c                 C  s"   | j dkrdS | jd | j  }|S )NrF   rk   ri   )re   r}   )r:   Zratio_of_suspicious_range_usager   r   r
   r_     s    
zSuspiciousRange.ratioNrm   r   r   r   r
   r{     s   r{   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )SuperWeirdWordPlugin_word_count_bad_word_count_foreign_long_count_is_current_word_bad_foreign_long_watchre   _bad_character_count_buffer_length_buffer_last_char_buffer_last_char_accentuated_buffer_accent_count_buffer_glyph_count_buffer_upper_countr5   r6   c                 C  sR   d| _ d| _d| _d| _d| _d| _d| _d| _d | _d| _	d| _
d| _d| _d S rg   r   r9   r   r   r
   r;     s    zSuperWeirdWordPlugin.__init__r<   r"   rW   c                 C  s  |j r|  jd7  _|| _|jr.|  jd7  _|j| _|jrJ|  jd7  _| jsh|j	r\|jrh|j
shd| _|j
r||  jd7  _dS | jsdS |js|jst|r|  jd7  _| j}|  j|7  _|dkr8| j| dkrd| _nV| jr| j r| j|kr|  jd7  _d| _n | jdkr8d| _|  jd7  _|dkr~| jr~| jdkob| j| dk}|s~|  jd7  _d| _| jr|  jd7  _|  j|7  _d	| _d	| _d| _d| _d	| _d| _d| _d| _n:|d
kr|js|jrd| _|  jd7  _|| _d	| _dS )rh   r   TN         ?   r   rl   F>   -=<|~>_)r&   r   r   r'   r   r.   r   r   r   r/   r2   r   r)   r3   r   r   re   r   rJ   r   r   r   r*   r4   )r:   r$   rX   Zbuffer_lengthZprobable_camel_casedr   r   r
   r\     s    
zSuperWeirdWordPlugin.feed_infoc                 C  sR   d| _ d | _d| _d| _d| _d| _d| _d| _d| _d| _	d| _
d| _d| _d S rg   )r   r   r   r   r   r   r   re   r   r   r   r   r   r9   r   r   r
   r]   ,  s    zSuperWeirdWordPlugin.resetr^   c                 C  s$   | j dkr| jdkrdS | j| j S )N
   r   rk   )r   r   r   re   r9   r   r   r
   r_   ;  s    zSuperWeirdWordPlugin.ratioNrm   r   r   r   r
   r     s   Kr   c                   @  sT   e Zd ZdZdZddddZdddd	d
dZddddZeddddZ	dS )CjkUncommonPluginz<
    Detect messy CJK text that probably means nothing.
    re   _uncommon_countr5   r6   c                 C  s   d| _ d| _d S rj   r   r9   r   r   r
   r;   K  s    zCjkUncommonPlugin.__init__r<   r"   rW   c                 C  s(   |  j d7  _ |tkr$|  jd7  _dS rq   )re   r   r   r[   r   r   r
   r\   O  s    zCjkUncommonPlugin.feed_infoc                 C  s   d| _ d| _d S rj   r   r9   r   r   r
   r]   V  s    zCjkUncommonPlugin.resetr^   c                 C  s.   | j dk rdS | j| j  }|dkr*|d S dS )Nr   rk   r   r   r   )r:   Zuncommon_form_usager   r   r
   r_   Z  s    
zCjkUncommonPlugin.ratioN)
rQ   rR   rS   rT   rU   r;   r\   r]   r`   r_   r   r   r   r
   r   C  s   r   c                   @  sP   e Zd ZdZddddZddddd	d
ZddddZeddddZdS )ArchaicUpperLowerPlugin	_buf_character_count_since_last_sep_successive_upper_lower_count#_successive_upper_lower_count_finalre   _last_alpha_seen_last_alpha_seen_upper_last_alpha_seen_lower_current_ascii_onlyr5   r6   c                 C  s:   d| _ d| _d| _d| _d| _d | _d| _d| _d| _d S )NFr   Tr   r9   r   r   r
   r;   t  s    z ArchaicUpperLowerPlugin.__init__r<   r"   rW   c                 C  s
  |j o
|j}| }|rv| jdkrv| jdkrF|jsF| jsF|  j| j7  _d| _d| _d| _d| _|  j	d7  _	d| _dS | jr|j
sd| _| jdk	r|jr| js|jr| jr| jr|  jd7  _d| _qd| _nd| _|  j	d7  _	|  jd7  _|| _|j| _|j| _dS )rh   r   @   NFr   Tri   )r&   r,   r   r*   r   r   r   r   r   re   r+   r'   r   r(   r   )r:   r$   rX   Zis_concernedZ	chunk_sepr   r   r
   r\     sJ    
z!ArchaicUpperLowerPlugin.feed_infoc                 C  s:   d| _ d| _d| _d| _d | _d| _d| _d| _d| _d S )Nr   FT)	re   r   r   r   r   r   r   r   r   r9   r   r   r
   r]     s    zArchaicUpperLowerPlugin.resetr^   c                 C  s   | j dkrdS | j| j  S )Nr   rk   )re   r   r9   r   r   r
   r_     s    
zArchaicUpperLowerPlugin.ratioNrm   r   r   r   r
   r   f  s   -r   c                   @  sP   e Zd ZdZddddZddddZdd	dd
ddZeddddZdS )ArabicIsolatedFormPluginre   _isolated_form_countr5   r6   c                 C  s   d| _ d| _d S rj   r   r9   r   r   r
   r;     s    z!ArabicIsolatedFormPlugin.__init__c                 C  s   d| _ d| _d S rj   r   r9   r   r   r
   r]     s    zArabicIsolatedFormPlugin.resetr<   r"   rW   c                 C  s*   |  j d7  _ |jt@ r&|  jd7  _dS rq   )re   r-   r   r   r[   r   r   r
   r\     s    
z"ArabicIsolatedFormPlugin.feed_infor^   c                 C  s   | j dk rdS | j| j  }|S )Nr   rk   r   )r:   Zisolated_form_usager   r   r
   r_     s    
zArabicIsolatedFormPlugin.ratioN)	rQ   rR   rS   rU   r;   r]   r\   r`   r_   r   r   r   r
   r     s   r      )maxsizez
str | NonerN   )r   r   r7   c                 C  s  | dks|dkrdS | |kr dS d| kr4d|kr4dS d| ksDd|krHdS d| ksXd|krld| kshd|krldS |  d| d }}|D ]}|tkrq||kr dS q| dk|dk }}|s|rd	| ksd	|krdS |r|rdS d
| ksd
|kr d	| ksd	|krdS | dks|dkr dS d	| ksHd	|ksH| dkr|dkrd| ks\d|kr`dS d| kstd|krxdS | dks|dkrdS dS )za
    Determine if two Unicode range seen next to each other can be considered as suspicious.
    NTFZLatinZ	EmoticonsZ	Combining )HiraganaKatakanaCJKZHangulzBasic Latin)r   r   ZPunctuationZForms)splitr   )r   r   Zkeywords_range_aZkeywords_range_belZrange_a_jp_charsZrange_b_jp_charsr   r   r
   r     sh    r   i   皙?Fr<   r^   )decoded_sequencemaximum_thresholddebugr7   c              	   C  s  t | }|dk rd}n|dk r$d}nd}t }t }t }t }t }	t }
t }t }t	 }|j
}|j
}|j
}|j
}|	j
}|
j
}|j
}|j
}|j
}t }|j}td||D ]}| |||  D ]}|| ||| ||| ||| |jr||| ||| |jr||| |jr(||| |jr:||| |jr||| q|j|j |j |j |	j |
j |j |j |j }||kr qq|d |d| |d| |d| |j|j |j |j |	j |
j |j |j |j }|rtd}|td	| d
| d|  |dkrX|td| dd   |td| dd   |||||	|
|||f	D ] }|t|j d|j  qnt|dS )zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    i  rD   r   r   r=   r   
Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=   zStarting with: NzEnding with: iz: r   )lenra   rn   rr   rv   r{   r   r   r   r   r\   r"   rP   ranger%   r&   r/   r0   r1   r_   r   logr   	__class__round)r   r   r   Zseq_lenstepZd_spZd_taZd_upZd_sdaZd_srZd_swZd_cuZd_auZd_aiZ	d_sp_feedZ	d_ta_feedZ	d_up_feedZ
d_sda_feedZ	d_sr_feedZ	d_sw_feedZ	d_cu_feedZ	d_au_feedZ	d_ai_feedrX   Zinfo_updateZblock_startr$   Zmean_mess_ratiologgerdtr   r   r
   
mess_ratio*  s    











r   )r   F)1
__future__r   sys	functoolsr   loggingr   version_infotypingr   Ztyping_extensionsImportErrorZconstantr   r   r   r   r   r   r   r   r   r   r   r   r   utilsr   r   r   r   r   r   r   r!   __annotations__r"   rV   ra   rn   rr   rv   r{   r   r   r   r   r   r   r   r   r   r
   <module>   sT    
<$ 1,7 "\I   