From fdb8dfeaf44d52097e6d9dff72e002d79c46933a Mon Sep 17 00:00:00 2001 From: acbaez9 <97056049+acbaez9@users.noreply.github.com> Date: Tue, 2 Jul 2024 19:23:35 -0700 Subject: [PATCH 1/4] working --- .DS_Store | Bin 0 -> 6148 bytes Dockerfile | 24 ++++++ filtered/__init__.py | 4 + filtered/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 236 bytes .../__pycache__/distributed.cpython-310.pyc | Bin 0 -> 2057 bytes filtered/__pycache__/filter.cpython-310.pyc | Bin 0 -> 8192 bytes filtered/__pycache__/split.cpython-310.pyc | Bin 0 -> 1086 bytes filtered/__pycache__/worker.cpython-310.pyc | Bin 0 -> 826 bytes filtered/distributed.py | 74 ++++++++++++++++++ filtered/filter.py | 6 +- filtered/worker.py | 23 ++++++ requirements.txt | 10 +++ 12 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 .DS_Store create mode 100644 Dockerfile create mode 100644 filtered/__init__.py create mode 100644 filtered/__pycache__/__init__.cpython-310.pyc create mode 100644 filtered/__pycache__/distributed.cpython-310.pyc create mode 100644 filtered/__pycache__/filter.cpython-310.pyc create mode 100644 filtered/__pycache__/split.cpython-310.pyc create mode 100644 filtered/__pycache__/worker.cpython-310.pyc create mode 100644 filtered/distributed.py create mode 100644 filtered/worker.py create mode 100644 requirements.txt diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..60b1127b0846d140ba84bc0f21cc76fe91ddfe30 GIT binary patch literal 6148 zcmeHKJ8r`;3?*9+2D)VIs4L_Kg5jK?7bsdcKmY><`q@3@Tsc}EKSKkzo3j{?0QDq_ zPlDcHnj)gz&#)KSiAW1Kl(P*>vwic0J!C|IaGdcWpYUP$dLI|re+P`)%27@-%J*;n z+cb@~NuvT(fC^9nDnJERRUpf2`?Tt*Jd6rZfh{Ot--iM>tci1=e>yOD3jk~ocEj9z z31G1RuqMudh`=g`kf;ze8v}hpx7{oyaOhAqU5ErWei4=wu#vFza2+atknV>W?kY);I&}4qe z2vn-cc#BcXPm?8zJ0-KYq$o3~v?Mj<7F$|oPDyIfE!N_KoXnD2Y~}ey*{MY<8H(6| zTEN6FJN?k&)S_bj#JrNE#MCN%m;B_?+|<01Vttp?)B;CO{ou@8m&B4pV;6mhVW}zl n@$s2?nI-Y@dIgoYIBatBQ%ZAE?Le+5<^U2L3>=Icj6940f?ztv literal 0 HcmV?d00001 diff --git a/filtered/__pycache__/distributed.cpython-310.pyc b/filtered/__pycache__/distributed.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1c0498e06ffca91aa9040c9d88044bbcae7c570d GIT binary patch literal 2057 zcmZuxNpBoQ6t1f2=^4-Fu|09d3vp)w<18~-4oD^nL|90<07>dmC%-dxp_ zIHMqeCf7jV1|gDtLXr3b-1z~V`-+h8jWYE3m>MFCtGwF|b3M+lIG;5-&kn zl(yLuDjTBAr^FPm038!v6%}3+RbCf0-Vk*@4b{EoKH)R?pM@IKp#jtT7N3KbXuvU; z8`CkP1b645xQ{+e5jZZIW6Imm7BkySez3E-b3+imfG1|hs4*aXQOt?fBLW4Og36d} z5Y%77{bPu@G^S6zeV0lb8$n7>i0ZVXVf$$Yr5bQ0DTHSNRJS@P9IQ|{m zXZ$2A?wx{@+LxVsqRmeu?t($H5pQu!#iCex%)adNGw`}t7O$Zni`s)RRe!=8;)L;R zBH^oWX75cvdUvf&G$yPHvTey<;k5cmOe1<}uY~zIJNci3b8xmz_<1-Ft4oAmfD5@* zSZ$LQ;T=Tk!g;as)Dq4!1|7qle5`X=+Pf%D85^(oCBOfe2-_kS8SBd0o8;B?CGv!- z(tY$#zn|czwe+`k`at*28qH>V61)`LzKlb+7e{@6a6t2|7VZf5u8JeqkK8RU?cD)| zM@n)Sxp(6&WUX{Zgg3LY>xLfcxT9s^rJjORb$ecNMYu>)KaNyK!}z}y2pN3)jedT7 z=ioA$87+HJy5+&B+xLSM({x=^z z#8Yh~pX+>-bUyaTKKTe+X@?1djYMxgr@Uyg{G7=us>*3ibhdI*M=qPXI)Dgv6M1bN zm&!uIKAR_U4w?U0*1M(>Eo4k3>_f|WHFDq?+?BJM{M(qx+1n^LIc$+pp>w&PQH24Q zAn+0eBAZJ65ZnOz>48r4+9kg5vL4=0c8)Hj*2wI<6{U7Av-1`+HITNJSQ=0dLgXQ^ z%Nu@_3{w{mftK(}UMA&_0I$-+(xWLp6?(gFBK;oZtm$pkiF-H*t{-{>;MFh|!vG98 zZw(-IB_N0%*a&)h!gy^W34AO&ujtuuCs6sE{Vr56n#qrT}!-h^G~VVl1+Vmc(V zy(Gbb2bZtrHMpkl4RsGm zjplEeRtgrgsLeFhZ<#e&gH{-0Wm=|;vcD|5jC(CMi+eTv%FNbsTK>CMv`}l-yiL?L a=d}j%O=__wwO>>Ut=}6j7(MgC`uQKzzbQEY literal 0 HcmV?d00001 diff --git a/filtered/__pycache__/filter.cpython-310.pyc b/filtered/__pycache__/filter.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..54e99740d7211641a11e05856f0e88c3f299119c GIT binary patch literal 8192 zcmai3>yO;lbsus#`_evSX(dat9n*3i+C*CM{cu#XZQ@C{YYtR{24ya7tm|*XZa#uLT`pI^OcVp?O8F$kM5}XE`O>$&yTei z`0@4;zS>?)8|@|DXfN{GvsxV$1tv|0YynPDDpGcGP}b<@7c zJ3YbEVei6SYu$`h>@xjPpSHh@qd0@6Y&=j#%0p$;=%`y@;18+Sl>Y$*E$=w-f7Rgef8Guo15*q8#ga+ymKAB#hbTY-`Kp~ zuCH2GzjbvJ!*J)VjT`OO+c!3FIs2`vZ(P58 zSJ$+hh-|MUdDiO;a#7|CEb(!xmvPY@&vaQNy2YA0)(X*0#tZUxr_Xxg(?)&-Q$M?O z@!dk?#YNUHBPObgSF(866aBKdctwc8#`TN0)7}+UGW*KKB<+?W7ktuLAKV+ybvkK3 zEjyhVl97gHQCU{2XFo^m>(il!WK2_9IHZgnbViLkjXRJs&MH^9`f+m)swOuWNkiRh zj+(`(QR6|AG(lI8ert2fmcKuH1*SA#Zs(%d&bmBSu;TPblA`Ty<0w|qY+ODSWm6d#mQIFOsVZj5IjjtT600sn$_9=pMyg>B;C^? zb(o_wQXarNxcX4Vxhc=e!^R$bX0M4uBTat$%wB8MdZ0c8yxmoDeWXF7ErOQ!kBzka z{HXb$$y*P!k@gUJR-mjsO+N9Td?GDiX7e-U{J1ez*4MPL!p17=x0~g*7&lp;f2Qbb zt?_Iw23;15ajO_~({kMEX1gLE&#=Kj^!Zr3lcoK&87a`56gqD0W;rj~iw7m%UXF)( z4k=GmFxF_$6GG09WBMr^5;ab#nxZLl$|C-ndK7*2LrtM`wf1?E{#;ET5;%bp9a7-O z(1ANZ=x_(MR7UE3AnT{f=9-$H#Itd;EBb5AnlCc@`~)?m_yfkg6vM2_sezuuL+Q;6 zpu2kE^Qb&D9sr(`Ypu-^A`rZEI^%^-2XHg&Qh%}2d4I^d^^<&w_9oh-pz^O!?=d~H zM;?|Fj|2|VK<64DrZu;;cvRkh#PQ&eRA?mm%8NAP*U@|jfIWgC-dDeWktFj#9jOnM z-)xl4?=>E14-xhDnq`Zt<;LguqiCH=m|XolQZC?eL0qXHE-DLfj8)~VqW=8s&+@P1=CzoNY5F5rB)v*& z{~dHjYOZmm4}jUzNW&xKN8k4WCu+V5NP+GMF#5A2wRlEe145KWsop`z;4MD$vC3!p z9BFiKhMtdR==rFbza&S`)76t2e)1X`zxoRrUx3DUbIKGd_16|E_58vk=@rOs(mn@h zJR)m&^mF_wbzu+aZQ>d@lcxx*q`z^C4|{`RZ8rCDm0zS=N0Kxz%36uzg)ZCgYZT4% z5M$bNQq@J7KQ^0M`p4zkooVQnCP9jl$RrVoPzfqHsRH{|MDL%d^JjmWU&EM3WK4%# zj8L99lB)2~DI52J>nU&@5Cn{e1Fifj)I!RDy8+Kf%8rts8nwukE;ZhtC$!GLgsFfe z^6zmAy!Q$T0zhBlxP7F@KA1w$c&-juSw3Fc=YpJI+=PAR63`xFhqSnrdR%7`6pJfZ zvsgwm15S@B%fK{3;QR(|#W?nV^TgJKC1lYsQG)1 zKR{H0k5JkL?}Fc{{Oyr8z5o86M7(bzxt$2ztX{nhpwi!Cd7&5R72D~cIIr)*5qfF2 zdtPU0j&R%;=k;yTP2+3;7r@;<*Lj|$yjb5~%c`E=(2*7CF%}%Kd7L!4+$rX$N0eW7>Rj?mM3cEsYHtsLX|;}x9h(uQ8LD7uI3O1((S zGVO2CeoWuZ;BbA(L$+H;F1lH^BcBh;BIRPetqbm&%r!$J6vFXhKQMS?v%nRBWhNrB zs#6>K76`+Fplf#<=C9{z(xthfX&RAdc!?LgjvrZJU?g#5$3g5w9y8-A0Utfq?ds7m z?Q+_9H!Zh8TApPIIE^V1uzm^&fq^K`cI#~-BtAQO0ZHGL5}C3RiGI92l~6rA%02)k zpgN2hV`jn($2BaLc%En4;LCGdPneaZp-Caxnd~DIr<$~y!ENx`MFeW= zR=^#bS!NLP!0|j5#AXs$U^VhhVVh1hn6kK?4GUODTEI&9d2q-eF5(t>Z~>y_Ngp8e zoY$iax;prf%B4whAe- zBu6+U!MKU#3fpCjGsh54XoimG`U%V_Hq9`sUfMUn1X>gIYJ;pO>vgtR2|*-+mBXA2 z10<7yMR-rZ>fE;-gSnC6MxNst35z2h8c=W&`H7jB)ibamcukb;UWl`9M();|coL>Z zk?p!xBErb<9Tq2nV6JV(mIwvt7Jl_dOqcqqsbV_MGV*4<6s)_yAQmUbCu8kR2S(i? zSb<5?1LW3svGMeBkY(L^9I^7mfI|yRx0?u;%mKrt-|ycdv$lQDVHT7uV&>SfA4S4* zcwmJ9PbW0o%HM!l4@9oVctzWR1xXjm@^p*!Qaws@A_MD|+rW^XC|S223Exc;+csU( zjs?n_GL7~49;y$B-$1rP^del4&dLn-Tux}9S6GpQ z?8Fa*9mH7K`IEL+3n3-hxPB%?T_{9*uKEVYl2p~7Z8m4K*h8tRe z6(^2sn#?d0ZuypJ3R*CzPVd_n;BV^Bgb$eP`-W`_(@J7Lal+ULUE5%s$8PM$X3XvC z%?-WweG*P>@;Ag*ruWnSor&QyDLz8d?G!Nq=0e;7{=wzMU>Lc zBRh29UJ2ulA3L5yu292ZO{wG+K?ZpeJQVURAV;`A(L$J)kh9ljht&<+w+#=0k~_j- zb{q-Q;il!siLksRa;s>%ddf#kNp3*ti7vWAmc3|w$4ETp!?|KOF7u*Lcy?^D*fv8mNQ~;*RJeD+HiKFZqV6Cg zs5+E-BrNaMCO8QnNh-Qz=m7$u9fh730hJwBgn`Y4V>--qeFmJW-XyDyckb;+!I-EA zWC%q2-CV?!3)Qhsim0FM-b0Gh=LPbx3CAF|D`+>^Hk{aEo^8aAYg=IqAcv_riQ#); z=v8MC(@_hAav+kvbV`}`)bMPy&(WhFDAg?G9wO! z?=#Z?B6w~X!VD~KaLWi0(yZ$I7C~lThpC7Q#WrMsR8bb)rnU*Rk@;VWtPxHanu+1q zrV*HqZ9Adwd!7wOT+6ULBd)@Ut2`Eay-tB55ao@8TU#Q7S_j)HN_rDZkeQArxEI)t z69cCqr#SGUfV-g^!%O`njl6$>AvG=!#MWe(qcjVjgP%?FJ&9g5Fv^VRD+RJ|M#cftLtG3H z7F^d6P>Jg@n7oOXi!ktkN8VIFNDyo1-pvrOBT-C6yOUCxp|6V%w%M>K0qhxHuf8}% zQQRTCp+u~nfoR`FDg|s?uW|;n&EZNZGMhmvVo7z%7`&hc!5pRg$iK^i7GEAU{wTAU~sw!X7o85=b!Mua+(=y@Qso%leV z4iI{+SF2MPsMnSG&w6!ay3f|IU;W8sQH0@p^`d2#mD^DcWm6(9uLej6h+Rn|&-z0` z#tB3d`%_}l{tjs?-%o8M%u!OMem~ex| z2I6cKPy`DSCxVlE30^y*IB;T5*j|WN83TD!jLgPCRwS5}AeK5nCkmquvcn`oSTT~Y zt;Y#Bfp?Z?#i8R`EDS8qO%m5cXhe?3kR2PKAm)@>!-qUTV&oxv0U`y+F)gR9dk!*9 z-?d^BZ|nh&Bf?t*aU`E`KL&hE4CFb6=i(IvK`lXOW4;k5ksmqS_G62RTkZK$p!y_Bkl&zjRDEe5=?{C;YR!}7r;~ER zq$V(_Lcyfp!o1>-(ab0+)x&?Tsp|h~P4#D*rv8uCQvX|QA&I>VIg9>W{T0_20E+_20A=^c41DY5%MpQ~yajuKuI8s{VsKPN;vcomBr$ z9#5-}v{UNeYNyq|QBSLi`XlWOj%U=r)}B@WN_$TIq4ov!FSX~jzfg5No%}(SGWX|4 zzq*Xt7;5!D%QrzTRkwS1OX_s0=hxRSk{}9!Pi0`x=KJ-gbt=YH=P+Peu-fm?m?uZ3 zKSre~{Zo`V-}!|?v8=$6ut?J1p+a$M4_~Wf6FP}TeuJld3k->(#Rp47wqr7aXc zJ{aUL{}!!AH9YmhV~zZq=*R^y^)mITR)%8!*XX2PyN2%T)O}i3C)+di7gPDsv9=T{ z*UDm{tfomJ<>zp3yqxw2!;(}k>&5MrY*-$=kp~#nK3bP2AJ>xeH|h3jnc>aqan(|O zY>)(wo5ipe}t4={Fg9W89Lb_VQHvlhWzTZ~%qXrR)RpIYJ!By5ZcV`g_I8 z&_|=Vgl3LDXEqnkEzB*f%+1ePn*5Z5&pGmZs;S{)=N!t3RKY%~u1@ZwxcDP&<^KSA Cat^!z literal 0 HcmV?d00001 diff --git a/filtered/__pycache__/split.cpython-310.pyc b/filtered/__pycache__/split.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c33c4b4c733a986c0645d74496e8f7b64d7e41a GIT binary patch literal 1086 zcmZWn&5P796i?c;&2&2MD1x70{Xm?{j^a&46vu-X5mpfkQo56jo85kGlW}#-pfGwA zPo6}WJ^C-@>S?du^rA1_6&Fm%%j?T~zx?uht$Mu>!T5aj?c@zX=)3(mJRpYdz><%^ zFvM_CT`H9a1PFb+P3!HSg%RS!V{=&t8vevEv z-2+tgaO~RcVDw<`z`6%Z{s5!UCSE$)*~PM}(9*RYMv7>_SU8`0b60swU%R__PM!ic z0@L17(rZY@JCJMVGgvCnWJ!0?0?$3w(H`u+3YT5&YkUSleV`%@Y)9z%tvS_H2glL8 zqdPh{f##v=X%GDHgLBFv02knOM8oSfW$p>|1u^o~-F4;>aEq7+-1?T==8EoWm-#IC z2(12NxP6%6_6>&C(11QPxB(3_m8E+aG?;m-o3bhm-r2Jq8DQg!lJQav`o^iGA$6uU zjL%;QsidJrw#AufAYRqHG^DL1d6h9ka!4_rP`r@F&FyJnYsr|4+;~k^zTk#1F;-Ak z2{>lcq8{{2D9U=O(uv5q>6Fu=^`z-lQ`N?+obkrs2^0YBW(8;_T~(@$>Hh_XOH=#AZ>TvBz`t$|pGe-wVOFa3jL(Ln zYCJ7K4mligUf+E%Toc6z5U!7gKv?mHv!SeWp;qhHCTzXbisAv7yaXn4LL9jfKKnaz zDL#TDhhmER_%uY1LDYFiel`~&(am9c+Dxx-vxZ=zino7(pFhP519)#iQXzIHqH5M X`}4`;U?{SJP=}r*4gq=k4*B^9YtJfe literal 0 HcmV?d00001 diff --git a/filtered/__pycache__/worker.cpython-310.pyc b/filtered/__pycache__/worker.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d589adda9ddcf1c78d63b0fdd91f5ab4e8978a28 GIT binary patch literal 826 zcmYjPO>fgc5S?8+iQ^bCBVTFvK;RuIQ191>$a&<36%Z| zgjA0GFZ_bNa_V2`iCISt8EHm-*6f?NZ?+r^e4zMqdp$ko0Dmpm4u`-orJ7O^AXo+= z2ritA5jv79w{U3$b|x`#6?@S>7OdzRI@#=k!SY0;3G z{X}AHmnh3TO3GrI&W1b2Q43|->x=z&dn4T%7ezF#SQwXPo|$LcdpOtza-RlCYlPX8 yIHPsZ6PngVV&4!^RF&MZ9oD6-(rEV%^HS6q9$Ax=CGy}A^PK1XK5MW2y#EIkDbRBO literal 0 HcmV?d00001 diff --git a/filtered/distributed.py b/filtered/distributed.py new file mode 100644 index 0000000..f282555 --- /dev/null +++ b/filtered/distributed.py @@ -0,0 +1,74 @@ + +from distributaur.distributaur import create_from_config +from .filter import read_json_in_batches +from .worker import run_job +from tqdm import tqdm +import time + + +if __name__ == "__main__": + + input_filename = 'datasets/cap3d_captions.json' + batch_size = 1000 + + distributaur = create_from_config() + + max_price = 0.1 + max_nodes = 50 + docker_image = "antbaez/filter-worker:latest" + module_name = "filtered.worker" + + redis_client = distributaur.get_redis_connection() + + rented_nodes = distributaur.rent_nodes(max_price, max_nodes, docker_image, module_name) + print("Total nodes rented: ", len(rented_nodes)) + + distributaur.register_function(run_job) + + while True: + user_input = input("press r when workers are ready: ") + if user_input == "r": + break + + + total_batches = 0 + + print("Sending tasks") + tasks = [] + + json_batches = [batch for batch in read_json_in_batches(input_filename, batch_size)] + print(f"number of batches: {len(json_batches)}") + + num_batches = len(json_batches) + for i in range(num_batches): + + batch = json_batches[i] + + total_batches += 1 + task = distributaur.execute_function("run_job", { + "batch_index" : total_batches, + "batch" : batch + }) + + tasks.append(task) + + first_task_done = False + print("Tasks sent. Starting monitoring") + with tqdm(total=len(tasks), unit="task") as pbar: + while not all(task.ready() for task in tasks): + current_tasks = sum([task.ready() for task in tasks]) + pbar.update(current_tasks - pbar.n) + if current_tasks > 0: + if not first_task_done: + first_task_done = True + first_task_start_time = time.time() + + end_time = time.time() + elapsed_time = end_time - first_task_start_time + time_per_tasks = elapsed_time / current_tasks + time_left = time_per_tasks * (len(tasks) - current_tasks) + + pbar.set_postfix( + elapsed=f"{elapsed_time:.2f}s", time_left=f"{time_left:.2f}" + ) + time.sleep(2) diff --git a/filtered/filter.py b/filtered/filter.py index 584960a..44ab912 100644 --- a/filtered/filter.py +++ b/filtered/filter.py @@ -18,7 +18,7 @@ def detect_objects(self, caption): if not objects: objects = self._extract_noun_phrases(caption) - print("These are the objects:", objects) + # print("These are the objects:", objects) return objects def _extract_noun_phrases(self, text): @@ -147,12 +147,12 @@ def test_caption_filtering(): if total_filtered_count >= write_batch_size or current_batch == total_batches: write_filtered_json(output_filename, filtered_data, first_batch=first_batch, last_batch=(current_batch == total_batches)) - print(f"Wrote batch {current_batch}/{total_batches} with {total_filtered_count} filtered captions") + # print(f"Wrote batch {current_batch}/{total_batches} with {total_filtered_count} filtered captions") filtered_data = {} total_filtered_count = 0 first_batch = False - print("Filtering and writing completed.") + # print("Filtering and writing completed.") # Optionally, you can keep the test function call if you want to run tests # test_caption_filtering() \ No newline at end of file diff --git a/filtered/worker.py b/filtered/worker.py new file mode 100644 index 0000000..96570af --- /dev/null +++ b/filtered/worker.py @@ -0,0 +1,23 @@ +import sys +from .filter import filter_captions, write_filtered_json + + +def run_job(batch_index, batch): + + output_filename = f"batch_{batch_index}" + + filtered_batch = filter_captions(batch) + write_filtered_json(output_filename, filtered_batch) + + distributaur.upload_file(output_filename) + + return "Task complete" + + +if __name__ == "__main__" or any("celery" in arg for arg in sys.argv): + from distributaur.distributaur import create_from_config + + distributaur = create_from_config() + distributaur.register_function(run_job) + + celery = distributaur.app diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..aea5435 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +requests +fsspec +celery +redis +huggingface_hub +python-dotenv +omegaconf +tqdm +gliner +distributaur From 22a857f17c25f4920c18bf2bb8203e1cc73bc4ba Mon Sep 17 00:00:00 2001 From: acbaez9 <97056049+acbaez9@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:40:18 -0700 Subject: [PATCH 2/4] filtering with distributask --- .DS_Store | Bin 6148 -> 0 bytes .../__pycache__/distributed.cpython-310.pyc | Bin 2057 -> 2423 bytes filtered/__pycache__/worker.cpython-310.pyc | Bin 826 -> 865 bytes filtered/distributed.py | 75 ++++++++++-------- filtered/worker.py | 18 +++-- requirements.txt | 2 +- 6 files changed, 55 insertions(+), 40 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 60b1127b0846d140ba84bc0f21cc76fe91ddfe30..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKJ8r`;3?*9+2D)VIs4L_Kg5jK?7bsdcKmY><`q@3@Tsc}EKSKkzo3j{?0QDq_ zPlDcHnj)gz&#)KSiAW1Kl(P*>vwic0J!C|IaGdcWpYUP$dLI|re+P`)%27@-%J*;n z+cb@~NuvT(fC^9nDnJERRUpf2`?Tt*Jd6rZfh{Ot--iM>tci1=e>yOD3jk~ocEj9z z31G1RuqMudh`=7fK6rMMXoz=!!J7Ed2^I!6V37CXZP(^5@DmNrlkpf~QWQ^xwEpgV)?wFsD z%2eTk1SQelQF3n8OAo#GRH+i;z=0YGam|4fAWkT}*^q$P)x3Q_-}~OTZ|-N_cR96= z4x7>Q>dkN0{;}7bKKOHOViwM?)qn5plLOd`16d{qF{}7VCbQ^bb_IY%1vu1HIn{fQ z;oe!c0&)b~*4s9i9K|H=!`up}KGpw_KYA?3a9j>)&oLb%6flxEjIZHyBkc9KnJb&9W?0EXG=j@8qVLKfH?^-pLY@$H-xt z_1!EMDiG^))tFg|{}m;$i)`^FF!Np_Z}H{Vn+X1nTt2%0>`m(4MgddFx%lMAqn~&q zof&7WVs41Br!$+`zn<|PfU)THo?`SuvCxOJT6j7&G|un82ZMhEB}yvdwAoH~%$e zi=L&9pJ~#NQBg&WL7ECnX(M-tq6N<{X;k?WGON5$2j10&5mxvz2#oGl^lRb*59g!* z6K_f6lVpF+q299gZg~23v0AH0oAwjvqV^V|&E!P0l?bbdb(tzRf&l%vfI)`1WU7L? zXe7jaDQYeT7}jfkh_annbR#U5soWA8Z|RmD)~lYbdOn8Is+K}sTnH;Ny(>Sa3cpy3 zSx|JR&22s)C&f}=UJGn4xPWAA1qiUnvp{S)du-0hvw#EtAQF@Zmv>Q}x=fy8n{X*% u@D8`A=0X?G5TnH%>a(~_6F4}~7oQa2GhB4uP0|?t&VA!-aTwVW|9u2@H3PW- delta 700 zcmZ8dJ8u&~5Z+y%&xf(^jN=5q@&G#+KVl#SNJNMx|9}QoNGC@-2PP&y?VbUJ0wkxQ z2*_Lo2;tCBKu1MIlOI6sf)0s@h60HK1!wLYiHI4^xAXYsq5YkHk}|u=WSpQp|FGpe zH3nvpE)43c)Ev0)r?Pww3NV)=d>(RwT_QZ^U!=81pu;re`&8sbVVga3co`N&QOs@; z$gh$`+ReBFMTXzd=du3X7K@HjM2MUORm2P={KU z@CG!XQXqT@mL^-_N+nA&gg0^2IcSLaSDILO&7c{2r?wf6jt$R?)8Uf;4wLWvw`}ob z>^>e4=nXWXUkj~T^1rg4^*ut!t(PwX`E+$}`Qut7OKHV_sM)?9wW;NQj?P+_@ynEk zz#h*KIpzO|uG~I>p-cx5{OAOdfoCGDpecrsA)ArrvNNKp%0@Bhv|I&Zbj)kt1-k=| zw-=DA7kEyzl_%S4J+IZ1&7*<;M!(5=(%;vA6)hx>^p@+whTxXdmddlemRh$(cf+q5 z_3M24NK$z%iFI7X4Q!ZriNhNvkr^K1f(*wfIfD!MaRr|3c9rL}AB>`^1K|3*MrrdN z787)$28~BFrcs>*7^uNg%%pL~m_ZH7DBIU`18W(U###c+V0w_J#@|F#LsmM}6tctq XfPsFBYAi+daXgawX^t5!kF_rc)+VhC diff --git a/filtered/__pycache__/worker.cpython-310.pyc b/filtered/__pycache__/worker.cpython-310.pyc index d589adda9ddcf1c78d63b0fdd91f5ab4e8978a28..7674a7bad03fe9c84721d6ebc3d4876892e47586 100644 GIT binary patch delta 423 zcmYjMu}T9$5Z#&G3zthylZb*~VUZ?QfOzPg;r_8CAUeABzMQ&9t63dkXk{m z6l?th!9Va9wzKyO1i{@)lZAQA8{WLP^I7^XxsmHSM8EgHr}4+L`*m+=LUmo}O3pP% zA|WGdSc4BU6gxg<(d0x5Vd4eT(6UrzqYpx#FO^l0et9zs(?vdoCRFlA zxI8raWJf~{t58A*ZFGP!;pRVZeO57UsIc8p)Dsh7_y(%FQ|0c2hkO}}-oB=Z@xq4# zA{u#ur{Lyq*jy&FwjU%?1jAJ7zl%MySlu3}y5JKn;bJgO!;uUkeUTT(VpmJIB8%`M c*X}MNKb`X)nAnB}o7BL@qk3@zILLng0jaZR@c;k- delta 342 zcmaFJwu_B7pO=@50SL~lY)NyN$a}J$1;}9rVrL*O)&deKj5Q3i8Rjz8GL1h zF{&_fO0hQBY(j2;>%n9KzHf#!w_Uc_NcBBhTccOe&J4MS93iD-r@30kUy2H?yQD aABcxA70lwEY{@Lm%Phmg!^j44&MyE=5l(pk diff --git a/filtered/distributed.py b/filtered/distributed.py index f282555..12caa63 100644 --- a/filtered/distributed.py +++ b/filtered/distributed.py @@ -1,38 +1,37 @@ - -from distributaur.distributaur import create_from_config +from distributask.distributask import create_from_config from .filter import read_json_in_batches from .worker import run_job from tqdm import tqdm import time - if __name__ == "__main__": - input_filename = 'datasets/cap3d_captions.json' - batch_size = 1000 + input_filename = "datasets/cap3d_captions.json" + batch_size = 10000 - distributaur = create_from_config() + distributask = create_from_config() - max_price = 0.1 - max_nodes = 50 + max_price = 0.25 + max_nodes = 25 docker_image = "antbaez/filter-worker:latest" module_name = "filtered.worker" - redis_client = distributaur.get_redis_connection() + redis_client = distributask.get_redis_connection() - rented_nodes = distributaur.rent_nodes(max_price, max_nodes, docker_image, module_name) + rented_nodes = distributask.rent_nodes( + max_price, max_nodes, docker_image, module_name + ) print("Total nodes rented: ", len(rented_nodes)) - distributaur.register_function(run_job) + distributask.register_function(run_job) while True: user_input = input("press r when workers are ready: ") if user_input == "r": break - total_batches = 0 - + print("Sending tasks") tasks = [] @@ -43,32 +42,44 @@ for i in range(num_batches): batch = json_batches[i] - total_batches += 1 - task = distributaur.execute_function("run_job", { - "batch_index" : total_batches, - "batch" : batch - }) + + print(total_batches) + task = distributask.execute_function( + "run_job", {"batch_index": total_batches, "batch": batch} + ) tasks.append(task) first_task_done = False print("Tasks sent. Starting monitoring") + + inactivity_log = {node["instance_id"]: 0 for node in rented_nodes} + + start_time = time.time() with tqdm(total=len(tasks), unit="task") as pbar: while not all(task.ready() for task in tasks): + current_tasks = sum([task.ready() for task in tasks]) pbar.update(current_tasks - pbar.n) - if current_tasks > 0: - if not first_task_done: - first_task_done = True - first_task_start_time = time.time() - - end_time = time.time() - elapsed_time = end_time - first_task_start_time - time_per_tasks = elapsed_time / current_tasks - time_left = time_per_tasks * (len(tasks) - current_tasks) - - pbar.set_postfix( - elapsed=f"{elapsed_time:.2f}s", time_left=f"{time_left:.2f}" - ) - time.sleep(2) + + time.sleep(1) + + current_time = time.time() + if current_time - start_time > 60: + start_time = time.time() + + for node in rented_nodes: + log_response = distributask.get_node_log(node) + if log_response.status_code == 200: + try: + last_msg = log_response.text.splitlines()[-1] + if ("Task complete" in last_msg and inactivity_log[node["instance_id"]] == 0): + inactivity_log[node["instance_id"]] = 1 + elif ("Task complete" in last_msg and inactivity_log[node["instance_id"]] == 1): + distributask.terminate_nodes([node]) + print("node terminated") + else: + inactivity_log[node["instance_id"]] == 0 + except: + pass diff --git a/filtered/worker.py b/filtered/worker.py index 96570af..c59cc18 100644 --- a/filtered/worker.py +++ b/filtered/worker.py @@ -1,23 +1,27 @@ import sys from .filter import filter_captions, write_filtered_json - def run_job(batch_index, batch): - output_filename = f"batch_{batch_index}" + if len(str(batch_index)) == 1: + batch_num = f"0{batch_index}" + else: + batch_num = f"{batch_index}" + + output_filename = f"batch_{batch_num}" filtered_batch = filter_captions(batch) write_filtered_json(output_filename, filtered_batch) - distributaur.upload_file(output_filename) + distributask.upload_file(output_filename) return "Task complete" if __name__ == "__main__" or any("celery" in arg for arg in sys.argv): - from distributaur.distributaur import create_from_config + from distributask.distributask import create_from_config - distributaur = create_from_config() - distributaur.register_function(run_job) + distributask = create_from_config() + distributask.register_function(run_job) - celery = distributaur.app + celery = distributask.app \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index aea5435..8b3b0ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ python-dotenv omegaconf tqdm gliner -distributaur +distributask From 1419555992ce220ff7e6ba2e2cf3a32d68cf29ad Mon Sep 17 00:00:00 2001 From: acbaez9 <97056049+acbaez9@users.noreply.github.com> Date: Wed, 10 Jul 2024 15:28:13 -0700 Subject: [PATCH 3/4] add gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..540914d --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.env +filtered/__pycache__ \ No newline at end of file From 13868b7d7f2ae690fec36cc93810263f07c59178 Mon Sep 17 00:00:00 2001 From: acbaez9 <97056049+acbaez9@users.noreply.github.com> Date: Wed, 10 Jul 2024 15:29:56 -0700 Subject: [PATCH 4/4] delete pycache --- filtered/__pycache__/__init__.cpython-310.pyc | Bin 236 -> 0 bytes filtered/__pycache__/distributed.cpython-310.pyc | Bin 2423 -> 0 bytes filtered/__pycache__/filter.cpython-310.pyc | Bin 8192 -> 0 bytes filtered/__pycache__/split.cpython-310.pyc | Bin 1086 -> 0 bytes filtered/__pycache__/worker.cpython-310.pyc | Bin 865 -> 0 bytes 5 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 filtered/__pycache__/__init__.cpython-310.pyc delete mode 100644 filtered/__pycache__/distributed.cpython-310.pyc delete mode 100644 filtered/__pycache__/filter.cpython-310.pyc delete mode 100644 filtered/__pycache__/split.cpython-310.pyc delete mode 100644 filtered/__pycache__/worker.cpython-310.pyc diff --git a/filtered/__pycache__/__init__.cpython-310.pyc b/filtered/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 532973d05e695f1c77e23be4ce644da9535f85b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 236 zcmd1j<>g`kf;ze8v}hpx7{oyaOhAqU5ErWei4=wu#vFza2+atknV>W?kY);I&}4qe z2vn-cc#BcXPm?8zJ0-KYq$o3~v?Mj<7F$|oPDyIfE!N_KoXnD2Y~}ey*{MY<8H(6| zTEN6FJN?k&)S_bj#JrNE#MCN%m;B_?+|<01Vttp?)B;CO{ou@8m&B4pV;6mhVW}zl n@$s2?nI-Y@dIgoYIBatBQ%ZAE?Le+5<^U2L3>=Icj6940f?ztv diff --git a/filtered/__pycache__/distributed.cpython-310.pyc b/filtered/__pycache__/distributed.cpython-310.pyc deleted file mode 100644 index 971c976c5f9ebbc25450f6ad763d96221e3cc385..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2423 zcmZWqOKclO7@irg9mlUYiJQiWX*O+|5}j-`>-zg|T87eSwVDM?E zi3*&6=>Z)uO7L_BvImG^oWMy@9Z)_Cvtn|W$s66x?lnPp9W|x~DA6W-PE3p1=L9k^ z4ut{RASgeN=O@tO{D6K(eqc&6-SGyz0*wLPAa@%46wJe^I^hdY7wjEv73x~HxNnLX zsKbId`8_*ed=cjMPs5_Z)V(KW`5CmYCB>YWN8i)h>k^FZk70agA&1`P22?b}D_^p& zf8@(>PMi`8C^@Gj2lO$GK7dz+sgVzPd|QQT2HN{Qc7~=+@b2^6`D*r66_ePZ5jHO69(IE z`;ewbH9Qg4P80@~8(3R*?A(P2RmNm5uy(>NB#pQ${Od{HvV0q5tbRk-u^mAiwH&*1 zQCPUfZWu&OWyYjYaU{X;{HU+1<_3T z{o2hiwmmZl1w^KVAO^8!-t3zl2@x{P`*$HQQ`yM0C7_Hg9;}%OmHqN21i}s4CL-L8 z92Mp9g9A)5_^w*c>mD+k(C=JcRwxvt zDv=fZ^rwTKzXj3^x6NT0)=Zhzcz98ButEa%WC?j6^%ric0L=&iG|kP}ma&T3#{$Hm zMDnH34OG*i&miHd(8pBhLUlL&Rwn=>FS zaGf-$cX6+SnjvyhE2uJ{*=(#z%8eevdkL8;ila1Vartb7Y@ZyEkFl$}Oc3m5di@FI zSzYQUOv8~=DE}gw@%1(Yu-B27-%}=4)a(($QBl9-d6bhHp6YhUshlX!X-lNm)u}w8 z7S$i=qC(e&yMly{xke7xkv{xYEl2qD%c@4EDz^vN_Y-)~pHL|@ap$`M_Pnp1>J1-0&OPAYTUfg!zsg7#~657x6Sdj#It&+ zc}ZVa!y(~CoSS2bBrENp4GDGil<+F-frG<0a-v%%$=RI_UK1Q|9vQ`!vTBJi0F^o$ z!Q6A>2bLGMc>!lZq5vkC8=Wm%CK=oFl1$X|dD-bn9jXSEICVxrMk+*64afHj4^I4MLO&y77dfQaWk8=?A9n@^0I zC(Y|~O{v>c4`qHmt=nZa!_gPGlt_uHU!E42!3>(yt3eH#M=DQ^qe`a07*cu4DEr6A zBA?dS6rQ7&;^V%6awS8_S6Cj;YIKTK(PExeP%qC4=z-A+ZRodz@>OO$%AptbSM6El Lh|#4ZyO;lbsus#`_evSX(dat9n*3i+C*CM{cu#XZQ@C{YYtR{24ya7tm|*XZa#uLT`pI^OcVp?O8F$kM5}XE`O>$&yTei z`0@4;zS>?)8|@|DXfN{GvsxV$1tv|0YynPDDpGcGP}b<@7c zJ3YbEVei6SYu$`h>@xjPpSHh@qd0@6Y&=j#%0p$;=%`y@;18+Sl>Y$*E$=w-f7Rgef8Guo15*q8#ga+ymKAB#hbTY-`Kp~ zuCH2GzjbvJ!*J)VjT`OO+c!3FIs2`vZ(P58 zSJ$+hh-|MUdDiO;a#7|CEb(!xmvPY@&vaQNy2YA0)(X*0#tZUxr_Xxg(?)&-Q$M?O z@!dk?#YNUHBPObgSF(866aBKdctwc8#`TN0)7}+UGW*KKB<+?W7ktuLAKV+ybvkK3 zEjyhVl97gHQCU{2XFo^m>(il!WK2_9IHZgnbViLkjXRJs&MH^9`f+m)swOuWNkiRh zj+(`(QR6|AG(lI8ert2fmcKuH1*SA#Zs(%d&bmBSu;TPblA`Ty<0w|qY+ODSWm6d#mQIFOsVZj5IjjtT600sn$_9=pMyg>B;C^? zb(o_wQXarNxcX4Vxhc=e!^R$bX0M4uBTat$%wB8MdZ0c8yxmoDeWXF7ErOQ!kBzka z{HXb$$y*P!k@gUJR-mjsO+N9Td?GDiX7e-U{J1ez*4MPL!p17=x0~g*7&lp;f2Qbb zt?_Iw23;15ajO_~({kMEX1gLE&#=Kj^!Zr3lcoK&87a`56gqD0W;rj~iw7m%UXF)( z4k=GmFxF_$6GG09WBMr^5;ab#nxZLl$|C-ndK7*2LrtM`wf1?E{#;ET5;%bp9a7-O z(1ANZ=x_(MR7UE3AnT{f=9-$H#Itd;EBb5AnlCc@`~)?m_yfkg6vM2_sezuuL+Q;6 zpu2kE^Qb&D9sr(`Ypu-^A`rZEI^%^-2XHg&Qh%}2d4I^d^^<&w_9oh-pz^O!?=d~H zM;?|Fj|2|VK<64DrZu;;cvRkh#PQ&eRA?mm%8NAP*U@|jfIWgC-dDeWktFj#9jOnM z-)xl4?=>E14-xhDnq`Zt<;LguqiCH=m|XolQZC?eL0qXHE-DLfj8)~VqW=8s&+@P1=CzoNY5F5rB)v*& z{~dHjYOZmm4}jUzNW&xKN8k4WCu+V5NP+GMF#5A2wRlEe145KWsop`z;4MD$vC3!p z9BFiKhMtdR==rFbza&S`)76t2e)1X`zxoRrUx3DUbIKGd_16|E_58vk=@rOs(mn@h zJR)m&^mF_wbzu+aZQ>d@lcxx*q`z^C4|{`RZ8rCDm0zS=N0Kxz%36uzg)ZCgYZT4% z5M$bNQq@J7KQ^0M`p4zkooVQnCP9jl$RrVoPzfqHsRH{|MDL%d^JjmWU&EM3WK4%# zj8L99lB)2~DI52J>nU&@5Cn{e1Fifj)I!RDy8+Kf%8rts8nwukE;ZhtC$!GLgsFfe z^6zmAy!Q$T0zhBlxP7F@KA1w$c&-juSw3Fc=YpJI+=PAR63`xFhqSnrdR%7`6pJfZ zvsgwm15S@B%fK{3;QR(|#W?nV^TgJKC1lYsQG)1 zKR{H0k5JkL?}Fc{{Oyr8z5o86M7(bzxt$2ztX{nhpwi!Cd7&5R72D~cIIr)*5qfF2 zdtPU0j&R%;=k;yTP2+3;7r@;<*Lj|$yjb5~%c`E=(2*7CF%}%Kd7L!4+$rX$N0eW7>Rj?mM3cEsYHtsLX|;}x9h(uQ8LD7uI3O1((S zGVO2CeoWuZ;BbA(L$+H;F1lH^BcBh;BIRPetqbm&%r!$J6vFXhKQMS?v%nRBWhNrB zs#6>K76`+Fplf#<=C9{z(xthfX&RAdc!?LgjvrZJU?g#5$3g5w9y8-A0Utfq?ds7m z?Q+_9H!Zh8TApPIIE^V1uzm^&fq^K`cI#~-BtAQO0ZHGL5}C3RiGI92l~6rA%02)k zpgN2hV`jn($2BaLc%En4;LCGdPneaZp-Caxnd~DIr<$~y!ENx`MFeW= zR=^#bS!NLP!0|j5#AXs$U^VhhVVh1hn6kK?4GUODTEI&9d2q-eF5(t>Z~>y_Ngp8e zoY$iax;prf%B4whAe- zBu6+U!MKU#3fpCjGsh54XoimG`U%V_Hq9`sUfMUn1X>gIYJ;pO>vgtR2|*-+mBXA2 z10<7yMR-rZ>fE;-gSnC6MxNst35z2h8c=W&`H7jB)ibamcukb;UWl`9M();|coL>Z zk?p!xBErb<9Tq2nV6JV(mIwvt7Jl_dOqcqqsbV_MGV*4<6s)_yAQmUbCu8kR2S(i? zSb<5?1LW3svGMeBkY(L^9I^7mfI|yRx0?u;%mKrt-|ycdv$lQDVHT7uV&>SfA4S4* zcwmJ9PbW0o%HM!l4@9oVctzWR1xXjm@^p*!Qaws@A_MD|+rW^XC|S223Exc;+csU( zjs?n_GL7~49;y$B-$1rP^del4&dLn-Tux}9S6GpQ z?8Fa*9mH7K`IEL+3n3-hxPB%?T_{9*uKEVYl2p~7Z8m4K*h8tRe z6(^2sn#?d0ZuypJ3R*CzPVd_n;BV^Bgb$eP`-W`_(@J7Lal+ULUE5%s$8PM$X3XvC z%?-WweG*P>@;Ag*ruWnSor&QyDLz8d?G!Nq=0e;7{=wzMU>Lc zBRh29UJ2ulA3L5yu292ZO{wG+K?ZpeJQVURAV;`A(L$J)kh9ljht&<+w+#=0k~_j- zb{q-Q;il!siLksRa;s>%ddf#kNp3*ti7vWAmc3|w$4ETp!?|KOF7u*Lcy?^D*fv8mNQ~;*RJeD+HiKFZqV6Cg zs5+E-BrNaMCO8QnNh-Qz=m7$u9fh730hJwBgn`Y4V>--qeFmJW-XyDyckb;+!I-EA zWC%q2-CV?!3)Qhsim0FM-b0Gh=LPbx3CAF|D`+>^Hk{aEo^8aAYg=IqAcv_riQ#); z=v8MC(@_hAav+kvbV`}`)bMPy&(WhFDAg?G9wO! z?=#Z?B6w~X!VD~KaLWi0(yZ$I7C~lThpC7Q#WrMsR8bb)rnU*Rk@;VWtPxHanu+1q zrV*HqZ9Adwd!7wOT+6ULBd)@Ut2`Eay-tB55ao@8TU#Q7S_j)HN_rDZkeQArxEI)t z69cCqr#SGUfV-g^!%O`njl6$>AvG=!#MWe(qcjVjgP%?FJ&9g5Fv^VRD+RJ|M#cftLtG3H z7F^d6P>Jg@n7oOXi!ktkN8VIFNDyo1-pvrOBT-C6yOUCxp|6V%w%M>K0qhxHuf8}% zQQRTCp+u~nfoR`FDg|s?uW|;n&EZNZGMhmvVo7z%7`&hc!5pRg$iK^i7GEAU{wTAU~sw!X7o85=b!Mua+(=y@Qso%leV z4iI{+SF2MPsMnSG&w6!ay3f|IU;W8sQH0@p^`d2#mD^DcWm6(9uLej6h+Rn|&-z0` z#tB3d`%_}l{tjs?-%o8M%u!OMem~ex| z2I6cKPy`DSCxVlE30^y*IB;T5*j|WN83TD!jLgPCRwS5}AeK5nCkmquvcn`oSTT~Y zt;Y#Bfp?Z?#i8R`EDS8qO%m5cXhe?3kR2PKAm)@>!-qUTV&oxv0U`y+F)gR9dk!*9 z-?d^BZ|nh&Bf?t*aU`E`KL&hE4CFb6=i(IvK`lXOW4;k5ksmqS_G62RTkZK$p!y_Bkl&zjRDEe5=?{C;YR!}7r;~ER zq$V(_Lcyfp!o1>-(ab0+)x&?Tsp|h~P4#D*rv8uCQvX|QA&I>VIg9>W{T0_20E+_20A=^c41DY5%MpQ~yajuKuI8s{VsKPN;vcomBr$ z9#5-}v{UNeYNyq|QBSLi`XlWOj%U=r)}B@WN_$TIq4ov!FSX~jzfg5No%}(SGWX|4 zzq*Xt7;5!D%QrzTRkwS1OX_s0=hxRSk{}9!Pi0`x=KJ-gbt=YH=P+Peu-fm?m?uZ3 zKSre~{Zo`V-}!|?v8=$6ut?J1p+a$M4_~Wf6FP}TeuJld3k->(#Rp47wqr7aXc zJ{aUL{}!!AH9YmhV~zZq=*R^y^)mITR)%8!*XX2PyN2%T)O}i3C)+di7gPDsv9=T{ z*UDm{tfomJ<>zp3yqxw2!;(}k>&5MrY*-$=kp~#nK3bP2AJ>xeH|h3jnc>aqan(|O zY>)(wo5ipe}t4={Fg9W89Lb_VQHvlhWzTZ~%qXrR)RpIYJ!By5ZcV`g_I8 z&_|=Vgl3LDXEqnkEzB*f%+1ePn*5Z5&pGmZs;S{)=N!t3RKY%~u1@ZwxcDP&<^KSA Cat^!z diff --git a/filtered/__pycache__/split.cpython-310.pyc b/filtered/__pycache__/split.cpython-310.pyc deleted file mode 100644 index 5c33c4b4c733a986c0645d74496e8f7b64d7e41a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1086 zcmZWn&5P796i?c;&2&2MD1x70{Xm?{j^a&46vu-X5mpfkQo56jo85kGlW}#-pfGwA zPo6}WJ^C-@>S?du^rA1_6&Fm%%j?T~zx?uht$Mu>!T5aj?c@zX=)3(mJRpYdz><%^ zFvM_CT`H9a1PFb+P3!HSg%RS!V{=&t8vevEv z-2+tgaO~RcVDw<`z`6%Z{s5!UCSE$)*~PM}(9*RYMv7>_SU8`0b60swU%R__PM!ic z0@L17(rZY@JCJMVGgvCnWJ!0?0?$3w(H`u+3YT5&YkUSleV`%@Y)9z%tvS_H2glL8 zqdPh{f##v=X%GDHgLBFv02knOM8oSfW$p>|1u^o~-F4;>aEq7+-1?T==8EoWm-#IC z2(12NxP6%6_6>&C(11QPxB(3_m8E+aG?;m-o3bhm-r2Jq8DQg!lJQav`o^iGA$6uU zjL%;QsidJrw#AufAYRqHG^DL1d6h9ka!4_rP`r@F&FyJnYsr|4+;~k^zTk#1F;-Ak z2{>lcq8{{2D9U=O(uv5q>6Fu=^`z-lQ`N?+obkrs2^0YBW(8;_T~(@$>Hh_XOH=#AZ>TvBz`t$|pGe-wVOFa3jL(Ln zYCJ7K4mligUf+E%Toc6z5U!7gKv?mHv!SeWp;qhHCTzXbisAv7yaXn4LL9jfKKnaz zDL#TDhhmER_%uY1LDYFiel`~&(am9c+Dxx-vxZ=zino7(pFhP519)#iQXzIHqH5M X`}4`;U?{SJP=}r*4gq=k4*B^9YtJfe diff --git a/filtered/__pycache__/worker.cpython-310.pyc b/filtered/__pycache__/worker.cpython-310.pyc deleted file mode 100644 index 7674a7bad03fe9c84721d6ebc3d4876892e47586..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 865 zcmYjP&2G~`5Z+xoiQ^{epFkiX#5I??h-+2xlM9z#3LGpU%kfTv6MJoT*KJcd5-2_M z0tg9i@C3ZeUODv@Dj_lJs3jxqr=8L4%=gV~(eL{N?dRVwli!SxKi1ha2c745)C7SO zPBRh`&bX7&oP{iRLWdIW^41jzU1;(4oT{0^V z6$yRtzi)y2gQD-xdSRc_Q>=eVhW<77A}7}ramdW}B+WF)D2Yp*7DAc*bD3(0)?wh$ zOcf$Q@^{w4nGpWh@&XRIc(=+mpQ1X#Ai({spKn8Mg(on|Tv4cb7#B$pBYv^GG z(>~u zc4A>%?0IG$@)Vbmjw_9eI@omNzGczd44GXCQ_RPoHmO8nzZ%vmOSz4y8~p7uWFx{m b$_rj)aBQcgEOn3UP>(&@ZAMFd=HC1ROIO>K