
    21jP\                       d Z ddlmZ ddlZddlZddlZddlmZmZmZm	Z	 ddl
mZ ddlmZ ddlmZmZmZ ddlmZ dd	lmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( dd
l)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5 ddl6m7Z7  ej8        e9          Z: G d d          Z;d'dZ<d(dZ=d)dZ>d*d Z?d+d,d&Z@dS )-u  The City Briefing Agent orchestrator.

One run = one HTTP trigger from n8n at 09:00 LA (or a manual POST).
For each show discovered `DAYS_TO_SHOW` days out, the orchestrator runs the
full flowchart:
    hub_lookup → previous_show_lookup → historical_sales_lookup →
    voice_profile_load → youtube_transcripts → perplexity_city → perplexity_news →
    openai_fact_check → openai_generation → claude_ranking →
    html_generation → vps_deployment → url_verification → og_verification →
    mark_complete (status='published')

Multi-show isolation: each show runs inside its own asyncio task. A failure
in one show's stage never cancels sibling shows. Concurrency is bounded by
`SHOW_CONCURRENCY`.
    )annotationsN)datedatetime	timedeltatimezone)Any)settings)PastShowShowVoiceProfile)
with_retry)STAGE_CLAUDE_RANKINGSTAGE_HUB_LOOKUPSTAGE_HISTORICAL_SALESSTAGE_HTML_GENERATIONSTAGE_OG_VERIFICATIONSTAGE_OPENAI_FACT_CHECKSTAGE_OPENAI_GENERATIONSTAGE_PERPLEXITY_CITYSTAGE_PERPLEXITY_NEWSSTAGE_PREVIOUS_SHOWSTAGE_URL_VERIFICATIONSTAGE_VPS_DEPLOYMENTSTAGE_VOICE_PROFILESTAGE_YOUTUBE_TRANSCRIPTSSTAGE_COLUMNSas_floatas_int
brief_slugredact_for_storagetoday_in_app_tz)AnthropicClient)	SccClient)OpenAIClient)PerplexityClient)YouTubeClient)
brief_reporun_repo)load_voice_profilec                  J    e Zd ZdZddZddZddZdddddZddZddZ	dS )OrchestrateBriefsServicezOwns a run's orchestration lifecycle and provider lifecycles.

    One instance per run. Not expected to be reused across runs.
    returnNonec                    t                      | _        t                      | _        t	                      | _        t                      | _        t                      | _	        d S N)
r#   sccr%   
perplexityr&   youtuber$   openair"   	anthropicselfs    @/workspace/show-city-briefing-agent/app/pipeline/orchestrator.py__init__z!OrchestrateBriefsService.__init__C   sA    ;;*,,$"nn(**    c                   K   | j         | j        | j        | j        | j        fD ]I}	 |                                 d {V  # t          $ r t                              dd           Y Fw xY wd S )Nzprovider close failedTexc_info)	r0   r1   r2   r3   r4   close	Exceptionloggerdebug)r6   clients     r7   aclosezOrchestrateBriefsService.acloseJ   s      x$,T^\ 	E 	EFEllnn$$$$$$$$ E E E4tDDDDDE	E 	Es   A&A*)A*run_idstrattemptinterrorBaseException | Nonec                V   K   |"|dk     rt          j        |           d {V  d S d S d S )N   )r(   increment_retry)r6   rC   rE   rG   s       r7   _record_retryz&OrchestrateBriefsService._record_retryQ   sK       1*622222222222 r9   scheduleN)triggerrun_daterN   rO   date | Nonedict[str, Any]c          
     l   K   |t                      }|t          t          j                  z   t                              d||                                                                           t          j                    }	 t           fdd           d{V }n# t          $ r}t                              d|           ddg t          j        t          j                                                  t          j        t          j                                                  t!          t          j                    |z
  d	z            d
dcY d}~S d}~ww xY wd |pg D             }t#          dt          j                  }t'          j        |          g }d fdt'          j        fd|D             ddi d{V }	|	D ]}
|                    |
           t!          t          j                    |z
  d	z            }t                              d|t/          |          |           t1          d |D                       rdn|rdnd|t          j        t          j                                                  t          j        t          j                                                  |dS )z}Discover shows DAYS_TO_SHOW out, then orchestrate each.

        Returns a summary dict suitable for the API caller.
        Ndaysz>orchestrator start: trigger=%s run_date=%s target_show_date=%sc                 8     j                                       S r/   )r0   find_shows_on_date)r6   targets   r7   <lambda>z.OrchestrateBriefsService.run.<locals>.<lambda>p   s    33F;; r9   
hub_lookup)
stage_namezhub_lookup run-level failed: %sfailed  u(   hub_lookup failed — no shows processed)statusbrief_run_idshows
started_atcompleted_atduration_msrG   c                ,    g | ]}t          |          S  )_to_show).0rs     r7   
<listcomp>z0OrchestrateBriefsService.run.<locals>.<listcomp>   s    IIIQXa[[IIIr9      showr   r,   rQ   c                   K   4 d {V                       |            d {V cd d d           d {V  S # 1 d {V swxY w Y   d S r/   )_orchestrate_one_show)rj   r6   sems    r7   	_per_showz/OrchestrateBriefsService.run.<locals>._per_show   s       > > > > > > > >!77========> > > > > > > > > > > > > > > > > > > > > > > > > > > > > >s   :
AAc              3  .   K   | ]} |          V  d S r/   rd   )rf   rj   rn   s     r7   	<genexpr>z/OrchestrateBriefsService.run.<locals>.<genexpr>   s+      00$iioo000000r9   return_exceptionsFz5orchestrator done: trigger=%s shows=%d duration_ms=%dc              3  H   K   | ]}|                     d           dk    V  dS )r]   	publishedN)get)rf   ss     r7   rp   z/OrchestrateBriefsService.run.<locals>.<genexpr>   s1      !S!SQ!%%//["@!S!S!S!S!S!Sr9   okpartial)r]   briefsr`   ra   rb   rj   r   r,   rQ   )r!   r   r	   DAYS_TO_SHOWr?   info	isoformattime	monotonicr   r>   rG   r   nowr   utcrF   maxSHOW_CONCURRENCYasyncio	Semaphoregatherappendlenall)r6   rN   rO   startedtour_shows_rawexcr_   concurrencyaccepted	summariesru   rb   rn   rm   rW   s   `           @@@r7   runzOrchestrateBriefsService.run^   s!      &((HI8+@AAAALX''))6+;+;+=+=	
 	
 	
 .""	#-;;;;;'$ $ $      NN  
	 
	 
	LL:C@@@" $&l8<88BBDD (X\ : : D D F F"DN$4$4w$>$#FGGF       
	 JI>3GRIII!X677,,)+	> 	> 	> 	> 	> 	> 	> ".0000%000
#
 
 
 
 
 
 
 
	  	 	AOOA4>++g5=>>CSZZ	
 	
 	

 "!S!S(!S!S!SSSxddfnYxYbYbtx",x|44>>@@$L66@@BB&
 
 	
s   	B% %
E/B$EEErj   r   c                >  K   t          j                    }t          j        |j        |j        |j        |j        |j        |j	                   d {V }t          |d                   }t          j        |j        |j                   d {V }t          |d                   }t          j        |t          d           d {V  t          j        |j                   d {V }|rt           t"          t$          t&          t(          t*          t,          t.          t0          t2          t4          t6          t8          f}|D ]}	t          j        ||	d           d {V  t          j        |dt=          t          j                    |z
  dz            d	           d {V  |j        |j        |j                                        d
|                     d          ||ddS t          j!        |           d {V  	 | "                    |||           d {V }
t=          t          j                    |z
  dz            }t          j#        ||           d {V  t          j        |t6          d           d {V  t          j        |t8          d           d {V  t          j        |d|           d {V  |j        |j        |j                                        d
|
                     d          ||dS # tH          $ r#}tJ          &                    d|j        |d           tO          |          }t          j(        ||           d {V  t          j)        |           d {V }|rCtT          D ];}	|                     |	          dk    r t          j        ||	d|           d {V   n<t          j        |dt=          t          j                    |z
  dz            |	           d {V  |j        |j        |j                                        d|||dcY d }~S d }~ww xY w)N)show_id	show_datecitycountryvenuecapacitybrief_id)r   r   r^   rv   skippedr\   zalready published)r]   rb   rG   rs   
public_urlT)r   r   r   r]   r   r   r^   reused)r]   rb   )r   r   r   r]   r   r   r^   z show %s orchestration failed: %sr;   runningr[   rG   )r   r   r   r]   r   r^   rG   )+r}   r~   r'   ensure_pendingr   r   r   r   r   r   rD   r(   insert	set_stager   fetch_existing_publishedr   r   r   r   r   r   r   r   r   r   r   r   r   mark_completerF   r|   rt   set_generating_execute_stagesmark_publishedr>   r?   warningr    mark_failedfetchr   )r6   rj   begin	brief_rowr   run_rowrC   existingremaining_stagesstagecontextrb   r   err_msgcurrent_runs                  r7   rl   z.OrchestrateBriefsService._orchestrate_one_show   s1       $3LnL*]
 
 
 
 
 
 
 
 
	 y,-- WWWWWWWWWW^,-- )94@@@@@@@@@ $<T\JJJJJJJJ 	 $%;=P)%'<')@$&;$&<%  * C C(	BBBBBBBBBB(tdn6F6F6NRV5V1W1W)         
  <	!^5577%&ll<88$ &	 	 	 '111111111,	 00xHHHHHHHHGt~//%74?@@K+HkBBBBBBBBB$V-CTJJJJJJJJJ$V-BDIIIIIIIII(+VVVVVVVVVV<	!^5577%%kk,77$ &    	 	 	NN=t|S[_N```(--G(7;;;;;;;;; (v 6 6666666K 	*  E"u--::&0"!$")	           ; (x!1!1E!9T ABB           <	!^5577"$ &        )	s    3C:K. .P9DPPPr   c                   K   |dt          j        t          d           d{V   j                            j        j                   d{V }|rt          |          ndt          j        t          rdnd           d{V  d<   t          j        t          d           d{V  i ra	 t           fd fdt          	           d{V n:# t          $ r-}t                              d
j        |           i Y d}~nd}~ww xY wr"t          j        t          d           d{V  n#t          j        t          dd           d{V  d<   t          j        t          d           d{V  t                      t          j        t          d           d{V  d<   t          j        t           d           d{V  	 t           fd fdt           	           d{V }t          j        t           d           d{V  nw# t          $ rj}t                              dj        |           g }t          j        t           dt#          t%          |                               d{V  Y d}~nd}~ww xY w|d<   t          j        t&          d           d{V  t           fd fdt&          	           d{V }t          j        t&          d           d{V  |                                d<   t          j        t*          d           d{V  rj        ndpj        t-          d          z
  t           fd fdt*          	           d{V }t          j        t*          d           d{V  d |j        D             d<   d |j        D             d<   t          j        t2          d           d{V  t           fd fdt2          	           d{V }	d  |	j        D             d!<   d" |	j        D             d#<   t          j        t2          d           d{V  t          j        t8          d           d{V  t           fd$ fd%t8          	           d{V }
d& |
D             d'<   t          j        t8          d           d{V  t          j        t:          d           d{V  t           fd( fd)t:          	           d{V }d* |D             d+<   d+         d,<   t          j        t:          d           d{V  t          j        t<          d           d{V  t?          j        j                  }t@          j!        "                    d-           d-| }d.d/l#m$}  |d         d         d!         d#         d,         d         %                    d          pg |0          }t          j        t<          d           d{V  |d1<   |d2<   |d3<   tM          j'        |d'         d+         d!         d#         d         d         d         d,         %                    d          pg rj        ndrj(        ndrj)        j)        ntU                    4           d{V  t          j        tV          d           d{V  tM          j,        ||           d{V  d.d5l-m.}  |||6           d{V  t          j        tV          d|7           d{V  d.d8l/m0} t          j        tb          d           d{V  t@          j2        rt@          j2        "                    d-          nt@          j!        "                    d-          }| d-| } ||           t          j        tb          d           d{V  t          j        tf          d           d{V  t@          j4        pd} ||d9|:           t          j        tf          d           d{V  S );u!  Run the per-show pipeline until completion. On exception, fails fast.

        Each stage marks itself running→ok/failed in `city_briefs_runs`.
        Each AI stage is wrapped in `with_retry` so transient provider errors
        retry the stage without aborting the whole show.
        )rj   r   rC   r   Nrv   r   previous_showc                 \    j                              j         j         j                  S )N)r   r   )r0   historical_salesr   r   r   )prevr6   s   r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>  s-    DH55"&.!Y 6   r9   c                2                         | |          S r/   rL   rE   rG   rC   r6   s     r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>  s    d6H6H7 7 r9   )
on_attemptrZ   z'historical_sales failed for show %s: %sz4no historical sales record matched the previous showr   r   voicec                 6     j                                         S r/   )r2   list_recent_with_transcriptsr5   s   r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>8  s    AACC r9   c                2                         | |          S r/   r   r   s     r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>9  s    $2D2DGU3 3 r9   uM   youtube_transcripts failed for show %s: %s — continuing without transcriptsrecent_transcriptsc                 \     j                             j        j        j                  S )N)r   r   r   )r1   city_intelligencer   r   r   )r6   rj   s   r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>J  s+    DO55Y 6   r9   c                2                         | |          S r/   r   r   s     r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>M      d.@.@/ / r9   r      rS   c                 ^     j                             j        j        j                  S )N)r   r   
since_dater   )r1   
local_newsr   r   r   )r6   rj   sinces   r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>Y  s.    DO..Y DN /   r9   c                2                         | |          S r/   r   r   s     r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>]  r   r9   c                6    g | ]}|                                 S rd   
model_dumprf   its     r7   rh   z<OrchestrateBriefsService._execute_stages.<locals>.<listcomp>c  s     III"bmmooIIIr9   recent_newsc                6    g | ]}|                                 S rd   r   r   s     r7   rh   z<OrchestrateBriefsService._execute_stages.<locals>.<listcomp>d  s     %Y%Y%Y"bmmoo%Y%Y%Yr9   competitor_warningsc                 x    j                             j        j        j         d          d                   S )Nr   r   )r   r   r   
city_intelr   )r3   
fact_checkr   r   r   )ctxr6   rj   s   r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>i  s<    DK**Y23. +   r9   c                2                         | |          S r/   r   r   s     r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>n  r   r9   c                6    g | ]}|                                 S rd   r   rf   fs     r7   rh   z<OrchestrateBriefsService._execute_stages.<locals>.<listcomp>s  s      T T TA T T Tr9   verified_factsc                6    g | ]}|                                 S rd   r   r   s     r7   rh   z<OrchestrateBriefsService._execute_stages.<locals>.<listcomp>t  s     RRR1RRRr9   removed_factsc                     j                             j        j        j         d          d          d         t                                         d          	  	        S )Nr   r   r   r   )	r   r   r   r   r   r   r   previous_show_summaryr   )r3   generate_twenty_ideasr   r   r   _previous_show_summaryrt   )r   r   salesr6   rj   r   s   r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>z  sf    DK55Y"#3423$'(=$>&<T5&I&I#&77+?#@#@ 6   r9   c                2                         | |          S r/   r   r   s     r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>  r   r9   c                6    g | ]}|                                 S rd   r   rf   is     r7   rh   z<OrchestrateBriefsService._execute_stages.<locals>.<listcomp>  s     "A"A"Aa1<<>>"A"A"Ar9   generated_topicsc                 n    j                             j        j         d          d                   S )Nr   r   )r   r   r   r   ideas)r4   rank_top_tenr   r   )r   r6   rj   r   s   r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>  s<    DN//Y$."#34,-	 0   r9   c                2                         | |          S r/   r   r   s     r7   rX   z:OrchestrateBriefsService._execute_stages.<locals>.<lambda>  r   r9   c                6    g | ]}|                                 S rd   r   r   s     r7   rh   z<OrchestrateBriefsService._execute_stages.<locals>.<listcomp>  s     ???1???r9   ranked_topics
crowd_work/r   )generate_brief_html)rj   r   r   r   r   r   r   r   r   r   r   r   htmlslugr   )r   r   r   r   r   r   r   r   r   previous_show_dateprevious_show_venueprevious_tickets_sold)deploy_brief_html)r   r   )r   )verify_public_urlT)
require_ogog_image_url)5r(   r   r   r0   find_previous_show_in_cityr   r   _to_past_showr   r   r>   r?   r   r   r   r)   r   _truncate_errrD   r   r   r   r   r   r   r   r   r   r   r   r   r   r	   PUBLIC_BASE_URLrstripapp.html.generatorr   rt   r'   set_generatedr   tickets_sold_sales_tickets_soldr   set_deployingapp.html.deployr   app.html.verifyr   r   INTERNAL_PUBLIC_BASE_URLr   INTERNAL_OG_COVER_URL)r6   rj   r   rC   	prev_showr   r   r   newsfact_resultr   rankedr   r   r   r   r   r   verification_baseverification_urlinternal_og_imager   r   r   r   r   s   `` `                 @@@@@r7   r   z(OrchestrateBriefsService._execute_stages   s      (,VTT  )<iHHHHHHHHH(==diXXXXXXXX	<E Oi 8 8 84 )<d>YddPYZZZZZZZZZ#O  )?KKKKKKKKK " 	(    
           6
 
 
 
 
 
 
 
 
    H$,X[\\\  	$V-CTJJJJJJJJJJ$&L	          #(  )<iHHHHHHHHH"$$ )<dCCCCCCCCCG  )BINNNNNNNNN	D'1CCCC     5( ( ( " " " " " " $V-FMMMMMMMMMM 	D 	D 	DNNj<. . .!#$V-F	+8S+B+BD D D D D D D D D D D D D D D		D %7 !  )>	JJJJJJJJJ%         -
 
 
 
 
 
 
 
 

  )>EEEEEEEEE#-#8#8#:#:   )>	JJJJJJJJJ#'1T[t~	WYHZHZHZ7Z          -	
 	
 	
 	
 	
 	
 	
 	
 	
  )>EEEEEEEEEII8HIIIM%Y%Y@X%Y%Y%Y!"  )@)LLLLLLLLL&     
     /

 

 

 

 

 

 

 

 

 !U T9S T T TRR8QRRRO )@$GGGGGGGGG  )@)LLLLLLLLL              /
 
 
 
 
 
 
 
 
 #B"A5"A"A"A )@$GGGGGGGGG  )=yIIIIIIIII!           ,
 
 
 
 
 
 
 
 
  @????O0L )=tDDDDDDDDD  )>	JJJJJJJJJ$)T^44 077<<EEtEE
::::::""./M*/0o.<( #$9 :""ww';<<B!
 
 
  )>EEEEEEEEEFF&L & !34o./0o.!"56M* #$9 :<("ww';<<B15?t~~4.2 <

%) 0d.?.K!!(//
 
 
 	
 	
 	
 	
 	
 	
 	
(  )=yIIIIIIIII&x<<<<<<<<<555555T5555555555 )=tPZ[[[[[[[[[[ 	655555 )?KKKKKKKKK 06H-44S999)0055 	
 088$88*+++ )?FFFFFFFFF  )>	JJJJJJJJJ$:Bd*	
 	
 	
 	

  )>EEEEEEEEE
s2    %C& &
D0#DD#AH) )
J3A JJ)r,   r-   )rC   rD   rE   rF   rG   rH   r,   r-   )rN   rD   rO   rP   r,   rQ   ry   )rj   r   r   rD   rC   rD   r,   rQ   )
__name__
__module____qualname____doc__r8   rB   rL   r   rl   r   rd   r9   r7   r+   r+   =   s         
+ + + +E E E E	3 	3 	3 	3 +5*.;
 ;
 ;
 ;
 ;
 ;
~\ \ \ \@j j j j j jr9   r+   rowrQ   r,   r   c                b   |                      d          }t          |t                    rt          j        |d d                   }n)t          |t
                    r|                                }t          t          |                      d          p+|                      d          p|                      d          pd          |t          |                      d          p|                      d          pd          t          |                      d	          p|                      d
          pd          |                      d          t          |                      d                    |                      d          p)|                      d          p|                      d                    S )Nr   
   r   show_keyid r   	city_namer   country_namer   r   
ticket_urltm_mainsale_url)r   r   r   r   r   r   r  )rt   
isinstancerD   r   fromisoformatr   r   r   r  raw_dates     r7   re   re     sd   ww{##H(C   #%hssm44	Hh	'	' #==??CGGI&&T#''**=*=TTRTUU>CGGK$8$8>B??CGGI&&G#''.*A*AGRHHggg
++,,77<((UCGGI,>,>U#''*BUBU   r9   r
   c                   |                      d          }t          |t                    rt          j        |d d                   }n)t          |t
                    r|                                }t          t          |                      d          p|                      d          pd          |t          |                      d          pd          t          |                      d          p|                      d          pd          |                      d	          t          |                      d
                    t          |                      d                    t          |                      d                              S )Nr   r  r   r  r  r   r   r  r   r   r   sold_pct)r   r   r   r   r   r   r   r!  )	rt   r  rD   r   r  r   r
   r   r   r  s     r7   r   r     s@   ww{##H(C   #%hssm44	Hh	'	' #==??CGGI&&C#''**=*=CDD&B''CGGI&&G#''.*A*AGRHHggg
++,,CGGN3344#''*--..	 	 	 	r9   r   PastShow | Noner   
str | Nonec                   | sd S | j          d| j                                         | j        pd g}| j        |                    | j         d           | j        |                    | j        dd           nf|                    d          *|                    d          |                    d          't          |          }|                    | d           d	                    |          S )	N z<venue unknown>z soldz.0%r   
total_soldtotalz; )
r   r   r|   r   r   r   r!  rt   r   join)r   r   partssolds       r7   r   r     s    t	88DN446688TZ=\K\:^_E$)000111} 00001111		.!!-99\"".99W)"5))^^^$$$99Ur9   
int | Nonec                `    dD ]*}t          |                     |                    }||c S +d S )N)r   r&  r'  )r   rt   )r   keyvalues      r7   r   r   !  sA    6  uyy~~&&LLL 4r9   r\   msgrD   limitrF   c                J    t          t          |                     d|         S )zCRedact + cap an error string for the city_briefs_runs error column.N)r    RuntimeError)r/  r0  s     r7   r   r   )  s!    l3//00%88r9   )r  rQ   r,   r   )r  rQ   r,   r
   )r   r"  r   rQ   r,   r#  )r   rQ   r,   r+  )r\   )r/  rD   r0  rF   r,   rD   )Ar  
__future__r   r   loggingr}   r   r   r   r   typingr   
app.configr	   
app.modelsr
   r   r   app.pipeline.retryr   app.pipeline.stagesr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   app.providers.anthropic_clientr"   app.providers.hubr#   app.providers.openai_clientr$   app.providers.perplexityr%   app.providers.youtuber&   app.repositoriesr'   r(   app.voice.loaderr)   	getLoggerr  r?   r+   re   r   r   r   r   rd   r9   r7   <module>rB     s     # " " " " "    8 8 8 8 8 8 8 8 8 8 8 8             3 3 3 3 3 3 3 3 3 3 ) ) ) ) ) )                                           , ; : : : : : ' ' ' ' ' ' 4 4 4 4 4 4 5 5 5 5 5 5 / / / / / / 1 1 1 1 1 1 1 1 / / / / / /		8	$	$j j j j j j j j^   "   $   $   9 9 9 9 9 9 9r9   