
    fZ                         d dl mZmZmZ d dlZd dlZd dlZd dlmZm	Z	 d dl
mZ d dlmZ d dlmZ  e       Z ej"                  e      Z G d de      Z G d d	      Z G d
 de      Zy)    )divisionprint_functionunicode_literalsN)unicode_strXDGDirs)Timer)Configc                       e Zd ZdZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Z	 	 	 	 	 	 	 ddZd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zy)WPLocalEnginezQ
    Singleton class for low-level word prediction, local in-process engine.
    c                     t        | d      s8t        j                  | g|i || _        | j                  j	                          | j                  S )z"
        Singleton magic.
        self)hasattrobject__new__r   	construct)clsargskwargss      2/usr/lib/python3/dist-packages/Onboard/WPEngine.pyr   zWPLocalEngine.__new__*   sA     sF#~~c;D;F;CHHH xx    c                      y)z4
        Called multiple times, do not use.
        N r   s    r   __init__zWPLocalEngine.__init__3   s     	r   c                     t               | _        t        | j                        | _        g | _        g | _        g | _        g | _        y)z8
        Singleton constructor, runs only once.
        N)
ModelCache_model_cacheAutoSaveTimer_auto_save_timermodelspersistent_modelsauto_learn_modelsscratch_modelsr   s    r   r   zWPLocalEngine.construct9   s@     'L -d.?.? @!#!# r   c                 l    | j                   j                          | j                  j                          y N)r   stopr   save_modelsr   s    r   cleanupzWPLocalEngine.cleanupD   s&    ""$%%'r   c                 P    ||z   | _         || _        || _        || _        || _        y)z" Fixme: rename to "set_model_ids" N)r    r!   r"   r#   )r   r!   r"   r#   s       r   
set_modelszWPLocalEngine.set_modelsH   s.    '.8!2!2!2,r   c                 N    | j                   j                  | j                         y)z
        Pre-load models set with set_models. If this isn't called,
        language models are lazy-loaded on demand.
        N)r   
get_modelsr    r   s    r   load_modelszWPLocalEngine.load_modelsP   s    
 	$$T[[1r   c                 8    | j                   j                          y r%   )r   postponer   s    r   postpone_autosavezWPLocalEngine.postpone_autosaveW   s    &&(r   c                 :    | j                   j                  d       y )N<   )r   pauser   s    r   pause_autosavezWPLocalEngine.pause_autosaveZ   s     	##B'r   c                 8    | j                   j                          y r%   )r   resumer   s    r   resume_autosavezWPLocalEngine.resume_autosave_   s    $$&r   c	                    t         j                  }	d}
|r|
|	j                  z  }
|r|
|	j                  z  }
|r|
|	j                  z  }
|r|
|	j
                  z  }
|r|
|	j                  z  }
|r|
|	j                  z  }
t        j                  |      \  }}| j                  | j                  |||
      }t        j                  dt        |      z          t        j                  dt        |dd       z          |D cg c]  }|d   	 c}S c c}w )z% Find completion/prediction choices. r   zcontext=zchoices=N   )	pypredictLanguageModelCASE_INSENSITIVECASE_INSENSITIVE_SMARTACCENT_INSENSITIVEACCENT_INSENSITIVE_SMARTIGNORE_CAPITALIZEDIGNORE_NON_CAPITALIZEDtokenize_context_get_predictionr    _loggerdebugrepr)r   context_linelimitcase_insensitivecase_insensitive_smartaccent_insensitiveaccent_insensitive_smartignore_capitalizedignore_non_capitalizedr;   optionscontextspanschoicesxs                  r   predictzWPLocalEngine.predictb   s     "//}555G!};;;G}777G#}===G}777G!};;;G"33LA&&t{{GUGLj4=01j4#445%&!&&&s   8Dc                    | j                   r<t        j                  |      \  }}t        |      D ]%  \  }}|j	                  d      s|dd }|sd}|||<   ' |r|g}n| j                  ||| j                        }| j                  j                  | j                         }|D ]  }	|D ]  }|	j                  |         t        j                  dt        |      z          t        j                  rXt        j                   j#                  t        j$                  d      }
t'        |
d      5 }|j)                  |dz          ddd       yyy# 1 sw Y   yxY w)	z7 Count n-grams and add words to the auto-learn models. 'Nz<unk>zlearn_text: tokens=zlearned_text.txta
)r"   r:   tokenize_text	enumerateendswith_drop_new_wordsr!   r   r,   learn_tokensrD   inforF   config	log_learnospathjoinuser_diropenwrite)r   textallow_new_wordstokensrQ   itokentoken_sectionsr    modelfnfs               r   
learn_textzWPLocalEngine.learn_text   sO   !!%33D9MFE &f- &5>>#&!#2JE  ' %F1I& "(!%!5!5fe6:6L6L"N&&11$2H2HIF /, /F&&v.// LL.n1EEF WW\\&//3EF"c] )aGGD4K() )  9 "<) )s   +EEc                     | j                  |||      \  }}t        |      D cg c]  \  }}t        d |D              r| }}}t        j                  ||      S c c}}w )z> Remove tokens that don't already exist in any active model.  c              3   &   K   | ]	  }|d k7    yw)   Nr   ).0ns     r   	<genexpr>z0WPLocalEngine._drop_new_words.<locals>.<genexpr>   s     =1Q=s   )lookup_tokensr[   allr:   split_tokens_at)	r   rj   rQ   lmidstokspanscountsrk   model_countssplit_indicess	            r   r]   zWPLocalEngine._drop_new_words   sh      --feUC&2;F2C ?q,===  ? ?((???s   Ac                     t        j                  |      \  }}| j                  j                  | j                        }|D ]  }|j                  |d        y)4 Count n-grams and add words to the scratch models. TN)r:   rZ   r   r,   r#   r^   )r   rh   rj   rQ   r    rn   s         r   learn_scratch_textz WPLocalEngine.learn_scratch_text   sQ    !//5""--d.A.AB 	-Evt,	-r   c                 |    | j                   j                  | j                        }|D ]  }|j                           y)r   N)r   r,   r#   clear)r   r    rn   s      r   clear_scratch_modelsz"WPLocalEngine.clear_scratch_models   s6    ""--d.A.AB 	EKKM	r   c                 X    t        j                  |      \  }}| j                  |||      S )z
        Split <text> into tokens and lookup the individual tokens in each
        of the given language models. See lookup_tokens() for more information.
        )r:   tokenize_sentencerx   )r   rh   r{   rj   rQ   s        r   lookup_textzWPLocalEngine.lookup_text   s-    
 "33D9!!&%77r   c                    t        |      D cg c]  \  }}||   d   ||   d   |f }}}|D cg c]  }|D cg c]  }d c} }}}t        |      D ]P  \  }}| j                  j                  |      }	|	s$t        |      D ]  \  }
}|	j                  |d         ||
   |<   ! R t        j                  dt        |      dt        |             ||fS c c}}w c c}w c c}}w )a|  
        Lookup the individual tokens in each of the given language models.
        This method is meant to be a basis for highlighting (partially)
        unknown words in a display for recently typed text.

        The return value is a tuple of two arrays. First an array of tuples
        (start, end, token), one per token, with start and end index pointing
        into <text> and second a two dimensional array of lookup results.
        There is one lookup result per token and language model. Each lookup
        result is either 0 for no match, 1 for an exact match or -n for
        count n partial (prefix) matches.
        r   rt      zlookup_tokens: tokens=z counts=)r[   r   	get_modellookup_wordrD   rE   rF   )r   rj   rQ   r{   rk   tr|   lmidr}   rn   js              r   rx   zWPLocalEngine.lookup_tokens   s     "+6!24A Ahqk58A;2 4 419:A'1':: ' 	;GAt%%//5E%h/ ;DAq#(#4#4QqT#:F1IaL;	; 	6lDL2 	3
 v~4':s   C	C&	C!C&!C&c                     d}| j                   }t        |      D ]>  \  }}| j                  j                  |      }|s$|j	                  |      }|dkD  s;d} |S  |S )zC
        Does word exist in any of the non-scratch models?
        Fr   T)r!   r[   r   r   r   )r   wordexistsr{   rk   r   rn   counts           r   word_existszWPLocalEngine.word_exists   sp     && ' 	GAt%%//5E))$/19!F	 r   c                 :    t        j                  |      \  }}||fS )z9
        Let the service find the words in text.
        )r:   rZ   )r   rh   rj   rQ   s       r   rZ   zWPLocalEngine.tokenize_text   s"     "//5u}r   c                 $    | j                  |      S )a9  
        Let the service find the words in text.
        Return python types instead of dbus.Array/String/... .

        Doctests:
        # whitspace have to be respected in spans
        >>> p = WPLocalEngine()
        >>> p.tokenize_text_pythonic("abc  def")
        (['abc', 'def'], [[0, 3], [5, 8]])
        )rZ   r   rh   s     r   tokenize_text_pythonicz$WPLocalEngine.tokenize_text_pythonic   s     !!$''r   c                 ,    t        j                  |      S )z( let the service find the words in text )r:   rB   r   s     r   rB   zWPLocalEngine.tokenize_context   s    ))$//r   c                 <    | j                   j                  |      }|S )z+ Return the names of the available models. )r   find_available_model_names)r   _classnamess      r   get_model_nameszWPLocalEngine.get_model_names  s    !!<<VDr   c                 f    |dd }| j                  |      \  }}t        |      r|d   d   }||d S y)z>
        Return the very last (partial) word in text.
        i NrW   r    )rB   len)r   rh   rj   rQ   begins        r   get_last_context_fragmentz'WPLocalEngine.get_last_context_fragment	  sH     EF|--d3u: "IaLE<r   c                 <   | j                   j                  |      \  }}| j                   j                  |      }|D ]4  }d|_        t	        |d      sd|_        d|_        d|_        g d|_        6 t        j                  |      }	|	j                  |||      }
|
S )Nzabs-discrecency_ratiogʡE?`   zjelinek-mercer)gB`"?gd;O?g"~j?)rO   )r   parse_lmdescr,   	smoothingr   r   recency_halfliferecency_smoothingrecency_lambdasr:   overlaypredictp)r   lmdescrP   rH   rO   r{   weightsr    mrn   rR   s              r   rC   zWPLocalEngine._get_prediction  s    **77?w""--e4 	:A %AK q/* #(%'"&6#$9!'	:* !!&) ..%.Ar   c           
      ,   | j                   j                  | j                        \  }}| j                   j                  |      }t	        |      D ]  \  }}|j                  |      }t        j                  dj                  |||   t        |                   t        j                  t        j                        smt        t        |j                               d       }|D ]-  }t        j                  dj                  |d   |d                /  y)z
        Remove the last word of context in the given context.
        If len(context) == 1 then all occurences of the word will be removed.
        z*removing {} from '{}': {} n-grams affectedc                      t        | d          S )Nr   )r   )rS   s    r   <lambda>z.WPLocalEngine.remove_context.<locals>.<lambda>G  s    AaD	z r   )keyz    remove: {}, count {}r   rt   N)r   r   r"   r,   r[   remove_contextrD   rE   formatr   isEnabledForloggingDEBUGsorteditems)	r   rP   r{   r   r    rk   r   changesngs	            r   r   zWPLocalEngine.remove_context8  s    
 **778N8NOw""--e4f% 	9DAq&&w/G MMF!6'58S\BD##GMM2 !8%9;! 9BMM"<#)6"Q%A#799	9r   N)   FFFFFF)__name__
__module____qualname____doc__r   r   r   r(   r*   r-   r0   r4   r7   rT   rq   r]   r   r   r   rx   r   rZ   r   rB   r   r   rC   r   r   r   r   r   r   %   s    	!(-2)(
' +-!&',#().#(',':!)F@-8:(0
B9r   r   c                       e Zd ZdZd Zd Zd Zd Zd Ze	d        Z
e	d        Ze	d	        Ze	d
        Ze	d        Zd Ze	d        Zd Ze	d        Zd Ze	d        Ze	d        Ze	d        Zy)r   z" Loads and caches language models c                     i | _         y r%   _language_modelsr   s    r   r   zModelCache.__init__P  
     "r   c                     i | _         y r%   r   r   s    r   r   zModelCache.clearS  r   r   c                 b    g }|D ]'  }| j                  |      }|s|j                  |       ) |S r%   )r   append)r   r{   r    r   rn   s        r   r,   zModelCache.get_modelsV  s:     	%DNN4(Ee$	% r   c                     | j                  |      }|| j                  v r| j                  |   }|S | j                  |      }|r|| j                  |<   |S )z3 get language model from cache or load it from disk)canonicalize_lmidr   
load_modelr   r   rn   s      r   r   zModelCache.get_model^  s_    %%d+4((())$/E
  OOD)E.3%%d+r   c                     g }| j                  |      }|D ]T  }t        j                  j                  |      }t        j                  j	                  |      \  }}|j                  |       V |S r%   )_find_modelsrb   rc   basenamesplitextr   )r   r   r   r    rn   nameexts          r   r   z%ModelCache.find_available_model_namesi  se    ""6* 	E77##E*D((.ID#LL	 r   c           	         g }| dk(  rt         j                         }nt         j                         }	 t        j                  |      }d}|D ]F  }|j                  d|z         s|j                  t        j                  j                  ||             H 	 |S # t        $ rX}t        j                  dj                  |t        j                  |j                        |j                               Y d }~|S d }~ww xY w)Nsystemlm.z/Failed to find language models in '{}': {} ({}))r`   get_system_model_dirget_user_model_dirrb   listdirr\   r   rc   rd   OSErrorrD   warningr   strerrorerrno)r   r    rc   files	extensionfilenamees          r   r   zModelCache._find_modelsr  s    X..0D,,.D	JJJt$EI! @$$S9_5MM"'',,tX">?@   	JOOM#VD"++agg*>HJ J	Js   0B #1B 	C8 AC33C8c                     g }g }| D ]W  }|j                  d      }|j                  |d          d}t        |      dk\  r	 t        |d         }|j                  |       Y ||fS #  Y xY w)zs
        Extract language model ids and interpolation weights from
        the language model description.
        ,r   g      ?r   rt   )splitr   r   float)r   r{   r   entryfieldsweights         r   r   zModelCache.parse_lmdesc  s      	#E[[%FLL#F6{a"6!9-F NN6"	# g~	s    A%%A)c                     g d}t        | j                  d      dd       D ]
  \  }}|||<    dj                  |      S )z
        Fully qualifies and unifies language model ids.
        Fills in missing fields with default values.
        The result is of the format "type:class:name".
        )r   r   en:N   )r[   r   rd   )r   resultrk   fields       r   r   zModelCache.canonicalize_lmid  sH     (!$**S/"1"56 	HAuF1I	xxr   c                 N    t         j                  |       } | j                  d      S )Nr   )r   r   r   )r   s    r   
split_lmidzModelCache.split_lmid  s     ++D1zz#r   c                 >    t         j                  |       \  }}}|dk(  S )Nuser)r   r   r   type_class_r   s       r   is_user_lmidzModelCache.is_user_lmid  s$    (33D9vtr   c                    |j                  d      \  }}}| j                  |      }|dk(  r|dk(  rBt        j                  |      dk(  rt        j                         }nt        j
                         }n|dk(  rt        j                         }nf|dk(  rt        j
                         }nLt        j                  dj                  ||             y t        j                  dj                  ||             y |r| j                  |||       |S )	Nr   r   r   rt   r   memz)Unknown class component '{}' in lmid '{}'z(Unknown type component '{}' in lmid '{}')r   get_filenamer:   
read_orderUnigramModelDynamicModelCachedDynamicModelrD   errorr   do_load_model)r   r   r   r   r   r   rn   s          r   r   zModelCache.load_model  s    #zz#vt$$T*D=!''1Q6%224E%224E6!!4465!..0I%vfd35MMD!6%.0uh7r   c                 4   t         j                  dj                  |             t        j                  j                  |      s+|dk(  r%t         j                  dj                  |             y y 	 | j                  |       y # t        $ r}|j                  0|j                  }t        j                  |      }t        d|||      }nt        |      }t         j                  |       || _        |dk(  rt         j                  d       Y d }~y Y d }~y d }~ww xY w)NzLoading language model '{}'.r   z3System language model '{}' doesn't exist, skipping.z+Failed to load language model '{}': {} ({})r   z>Saving word suggestions disabled to prevent further data loss.)rD   r_   r   rb   rc   r   r   loadIOErrorr   r   _formatr   r   load_error_msg)rn   r   r   exr   errstrmsgs          r   r  zModelCache.do_load_model  s    3::8DEww~~h'! !;!'!13 "
C

8$ C88'HHE[[/F!I$fe5C &b/Cc"'*$V#MM #B C C $Cs   0B 	DA=DDc                     t        | j                  j                               D ])  \  }}| j                  |      s| j	                  ||       + y r%   )listr   r   can_save
save_modelr   s      r   r'   zModelCache.save_models  sC     5 5 ; ; => 	-KD%}}T"t,	-r   c                 6    | j                  d      \  }}}|dk(  S )Nr   r   )r   r   s       r   r  zModelCache.can_save  s!    #zz#vtr   c           	         |j                  d      \  }}}| j                  |      }| j                  |      }|r$|j                  r|j                  r%t
        j                  dj                  |             y t
        j                  dj                  |             	 t        j                  j                  |      }t        j                  |       	 t        j                  j                  |      \  }	}
|	dz   }|j                  |       t        j                  j!                  |      rt        j"                  ||       t        j"                  ||       d|_        y y y # t$        t&        f$ rW}t
        j                  dj                  |t        j(                  |j*                        |j*                               Y d }~y d }~ww xY w)Nr   zFNot saving modified language model '{}' due to previous error on load.zSaving language model '{}'z.tmpFz+Failed to save language model '{}': {} ({}))r   r   get_backup_filenamemodified
load_errorrD   r   r   r_   rb   rc   dirnamer   assure_user_dir_existsr   saver   renamer  r   r   r   )r   rn   r   r   r   r   r   backup_filenamerc   r   r   tempfiler   s                r   r  zModelCache.save_model  sc   #zz#vt$$T*228<>> !A!'!13 9@@JKJ77??84D2248(*(8(8(B##+f#4

8, 77>>(3IIh@		(H5%*EN1  4  ) JOOE"++agg*>HJ JJs   B?E   G/AGGc                     | j                  d      \  }}}|dk(  rd}|S |dk(  rt        j                         }nt        j                         }|}t        j
                  j                  ||dz   |z         }|S )Nr   r   r   r   r   )r   r`   r   r   rb   rc   rd   )r   r   r   r   r   rc   r   s          r   r   zModelCache.get_filename  sv    #zz#vtU?H  !224002Cww||D$*s*:;Hr   c                     | dz   S )Nz.bakr   )r   s    r   r  zModelCache.get_backup_filename,  s    &  r   c                     d}	 dj                  | t        j                  d      |      }t        j                  j                  |      s	 |S |dz  }N)a  
        Filename broken files are renamed to.

        Doctests:
        >>> import tempfile
        >>> import subprocess
        >>> from os.path import basename
        >>> td = tempfile.TemporaryDirectory(prefix="test_onboard_")
        >>> dir = td.name
        >>> fn = os.path.join(dir, "en_US.lm")
        >>>
        >>> def test(fn):
        ...     bfn = ModelCache.get_broken_filename(fn)
        ...     print(repr(basename(bfn)))
        ...     _ignore = subprocess.call(["touch", bfn])

        >>> test(fn)   # doctest: +ELLIPSIS
        'en_US.lm.broken-..._001'

        >>> test(fn)   # doctest: +ELLIPSIS
        'en_US.lm.broken-..._002'

        >>> test(fn)   # doctest: +ELLIPSIS
        'en_US.lm.broken-..._003'
        rt   z{}.broken-{}_{:03}z%Y-%m-%d)r   timestrftimerb   rc   r   )r   r   ro   s      r   get_broken_filenamezModelCache.get_broken_filename0  sW    6 %,,X-1]]:-F-24B 77>>"%	 QJE r   N)r   r   r   r   r   r   r,   r   r   staticmethodr   r   r   r   r   r   r  r'   r  r  r   r  r  r   r   r   r   r   M  s    ,##	  &  . 
  
       : C C6-
    #JJ   ! ! " "r   r   c                   8    e Zd ZdZ	 	 	 ddZd	dZd Zd Zd Zy)
r   z1 Auto-save modified language models periodically c                     || _         || _        || _        || _        | j                  | _        t        j
                         | _        d| _        d| _        | j                  | j                  | j                         y )Nr   r9   )r   _interval_min_interval_max_postpone_delay	_intervalr  _last_save_time_pause_timer_intervalstart	_on_timer)r   
mode_cacheinterval_mininterval_maxpostpone_delays        r   r   zAutoSaveTimer.__init__Y  sf     '))-++#yy{ 

4''8r   Nc                     || _         y)zE
        No auto-saving while paused, e.g. during key-press.
        Nr'  )r   durations     r   r3   zAutoSaveTimer.pauseg  s     r   c                     d| _         y)z*
        Allow auto-saving again.
        r   Nr0  r   s    r   r6   zAutoSaveTimer.resumem  s     r   c                 X   t        j                          | j                  z
  }| j                  || j                  z   k  r>|| j                  z   | _        | j                  | j                  kD  r| j                  | _        t
        j                  dj                  | j                  |             y)z
        Postpone saving a little while the user is still typing.
        Helps to mask the delay when saving large models, during which
        Onboard briefly becomes unresponsive.
        zDpostponing autosave: current interval {}, elapsed since last save {}N)r  r&  r%  r$  r#  rD   rE   r   )r   elapseds     r   r/   zAutoSaveTimer.postpones  s     ))+ 4 44>>Gd&:&:::$t';';;DN~~ 2 22!%!3!3 3vdnng6	8r   c                    t        j                          }|| j                  z
  }| j                  |k  rp| j                  dk(  ra|| _        | j                  | _        t
        j                  dj                  | j                  |             | j                  j                          | j                  r(t        d| j                  | j                  z
        | _        y)Nr   z9auto-saving language models; interval {}, elapsed time {}T)r  r&  r%  r'  r"  rD   rE   r   r   r'   maxr(  )r   nowr4  s      r   r*  zAutoSaveTimer._on_timer  s    iik,,,>>G#;;!#&D !//DNMM 9!6$..':< ))+;;at/C/C!CDDKr   )iX  i  
   r%   )	r   r   r   r   r   r3   r6   r/   r*  r   r   r   r   r   V  s)    ; &% "98r   r   )
__future__r   r   r   rb   r  r   Onboard.utilsr   r   Onboard.Timerr   Onboard.Configr	   Onboard.pypredictr:   r`   	getLoggerr   rD   r   r   r   r   r   r   r   <module>r?     sd   ( B A 	   .  ! %	
'

H
%e9F e9P	F FR;E ;r   