
    ]Di                        S SK Jr  S SKrS SKrS SKrS	S jrS
S jrSSS jjr   S           SS jjrSS jr	SSS jjr
g)    )annotationsNc                   SU ;  a%  [        U SSS9 nUR                  5       sS S S 5        $ U R                  S5      (       a1  SS KnUR	                  U 5      nUR                  5         UR                  $  SS KnUR                  U S5       nUR                  5       sS S S 5        $ ! , (       d  f       N= f! [         a  n[        S5      UeS nAff = f! , (       d  f       g = f)Nz://rbr   	buffering)zhttp://zhttps://Oblobfile is not installed. Please install it by running `pip install blobfile`.)
openread
startswithrequestsgetraise_for_statuscontentblobfileImportErrorBlobFile)blobpathfr   respr   es         U/var/www/html/Aiprofessor/professorTrac/lib/python3.13/site-packages/tiktoken/load.py	read_filer      s    H(DA.!668 /. 233||H%||
 
		8T	*avvx 
+	*# /.  ]
	 
+	*s/   B$4B5 
C$
B25
C?CC
C!c                R    [         R                  " U 5      R                  5       nX!:H  $ N)hashlibsha256	hexdigest)dataexpected_hashactual_hashs      r   
check_hashr!      s"    ..&002K''    c                   SnS[         R                  ;   a  [         R                  S   nO\S[         R                  ;   a  [         R                  S   nO4SS Kn[         R                  R	                  UR                  5       S5      nSnUS:X  a  [        U 5      $ [        R                  " U R                  5       5      R                  5       n[         R                  R	                  X55      n[         R                  R                  U5      (       aQ  [        USSS	9 nUR                  5       nS S S 5        Ub  [        WU5      (       a  W$  [         R                  " U5        [        U 5      n	U(       a"  [        X5      (       d  [#        S
U  SU S35      eSS Kn
 [         R&                  " USS9  US-   [)        U
R+                  5       5      -   S-   n[        US5       nUR-                  U	5        S S S 5        [         R.                  " X5        U	$ ! , (       d  f       N= f! [          a     Nf = f! , (       d  f       NG= f! [          a    U(       a  e  U	$ f = f)NTTIKTOKEN_CACHE_DIRDATA_GYM_CACHE_DIRr   zdata-gym-cacheF r   r   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)exist_ok.z.tmpwb)osenvirontempfilepathjoin
gettempdirr   r   sha1encoder   existsr	   r
   r!   removeOSError
ValueErroruuidmakedirsstruuid4writerename)r   r   user_specified_cache	cache_dirr,   	cache_key
cache_pathr   r   contentsr6   tmp_filenames               r   read_file_cachedrB   $   s   rzz)JJ34			+JJ34	GGLL!4!4!68HI	$B""X__./99;Ii3J	ww~~j!!*da0A668D 1 Jt]$C$CK	IIj! "HZ@@5hZ{=/ ZH I
 	

 	
I-!C'#djjl*;;fD,%GGH &
		,+ O? 10  		 &%     OsI   HH AH> H-,H> 
H
H*)H*-
H;7H> >IIc                ^  ^ [        S5       Vs/ s H6  n[        U5      R                  5       (       d  M#  [        U5      S:w  d  M4  UPM8     nnU Vs0 s H  n[        U5      U_M     snmSn[        S5       H1  nXV;  d  M
  UR                  U5        UT[        SU-   5      '   US-  nM3     [	        U5      S:X  d   e[        X5      R                  5       nUR                  S5      SS  V	s/ s H  n	[        U	R                  5       5      PM     n
n	S
U4S jjn[        U5       VVs0 s H  u  p[        U/5      U_M     nnnA[	        U5      nU
 H  u  pX}U" U5      U" U5      -   '   US-  nM     SS KnUR                  [        X5      5      nUR                  5        VVs0 s H  u  nnU" U5      U_M     nnnUR                  SS 5        UR                  S	S 5        U(       a"  U H  n[	        U5      S:X  d  M  UU   UU'   M     UU:X  d   eU$ s  snf s  snf s  sn	f s  snnf s  snnf )N    r      
c                .   > [        U4S jU  5       5      $ )Nc              3  .   >#    U  H
  nTU   v   M     g 7fr    ).0bdata_gym_byte_to_bytes     r   	<genexpr>Kdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>r   s     =u!*1-us   )bytes)valuerN   s    r   decode_data_gym8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymq   s    =u===r"   s   <|endoftext|>s   <|startoftext|>)rR   r8   returnrQ   )rangechrisprintableappendlenrB   decodesplittuple	enumeraterQ   jsonloadsitemspop)vocab_bpe_fileencoder_json_filevocab_bpe_hashencoder_json_hashclobber_one_byte_tokensrM   rank_to_intbytenvocab_bpe_contents	merge_str
bpe_mergesrS   i	bpe_ranksfirstsecondr_   encoder_jsonkvencoder_json_loadedrN   s                        @r   data_gym_to_mergeable_bpe_ranksru   Z   s(    #(+X+QQ1C1C1Eq#a&TW-q+OX0?@1SVQY@	A4[#""1%34!#dQh-0FA	 
 4''' *.IPPR<N<T<TUY<Z[\]_<`a<`y%	)*<`Ja>
 ,5_+EF+E41sQ+EIF 	IA#FG/%(?6+BBC	Q $ 
 ::./@TUL=I=O=O=QR=QTQ?1-q0=QR,d3.5$A1v{215	! % ++++_ Y@ b G Ss(   "HHHH-#H(H#$H)c                p    SS K nUR                  US5       n[        U R	                  5       S S9 HJ  u  pVUR                  [        R                  " U5      S-   [        U5      R                  5       -   S-   5        ML     S S S 5        g ! [         a  n[        S5      UeS nAff = f! , (       d  f       g = f)Nr   r   r)   c                    U S   $ )NrF   rK   )xs    r   <lambda>#dump_tiktoken_bpe.<locals>.<lambda>   s    1Q4r"   )key       
)
r   r   r   sortedra   r:   base64	b64encoder8   r1   )rn   tiktoken_bpe_filer   r   r   tokenranks          r   dump_tiktoken_bper      s    
 
		,d	3q!)//"3HKEGGF$$U+d2SY5E5E5GG%OP I 
4	3	  ]
	 
4	3s#   B	 A(B'	
B$BB$'
B5c                   [        X5      n0 nUR                  5        HA  nU(       d  M   UR                  5       u  pV[        U5      U[        R
                  " U5      '   MC     U$ ! [         a  n[        SU< SU  35      UeS nAff = f)NzError parsing line z in )rB   
splitlinesr\   intr   	b64decode	Exceptionr5   )r   r   r@   retliner   r   r   s           r   load_tiktoken_bper      s     1AH
C##%	[**,KE+.t9C  '( & J  	[24($?P>QRSYZZ	[s   4A%%
B/BB)r   r8   rU   rQ   )r   rQ   r   r8   rU   boolr   )r   r8   r   
str | NonerU   rQ   )NNF)rc   r8   rd   r8   re   r   rf   r   rg   r   rU   dict[bytes, int])rn   r   r   r8   rU   None)r   r8   r   r   rU   r   )
__future__r   r   r   r*   r   r!   rB   ru   r   r   rK   r"   r   <module>r      su    "   	.(
3r "&$($)777 7 "	7
 "7 7t	Qr"   