From 8c3ffca0bc30a39d9c3bfe5ca167ca329cc85511 Mon Sep 17 00:00:00 2001 From: olender Date: Mon, 1 Jul 2024 12:38:53 -0300 Subject: [PATCH] variable name change --- .vscode/launch.json | 28 ++++----- spyro/io/basicio.py | 11 ++-- spyro/io/model_parameters.py | 11 +++- spyro/solvers/acoustic_wave.py | 1 + spyro/utils/utils.py | 12 ++-- temp_forward_shot.py | 13 ++-- temp_test_parallel.py | 110 +++++++++++++++++++++++++++++++++ velocity_models/tutorial | Bin 518568 -> 0 bytes 8 files changed, 151 insertions(+), 35 deletions(-) create mode 100644 temp_test_parallel.py delete mode 100644 velocity_models/tutorial diff --git a/.vscode/launch.json b/.vscode/launch.json index 99d93959..376f4a95 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,20 +4,20 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ - // { - // "name": "Python Attach 0", - // "type": "python", - // "request": "attach", - // "port": 3000, - // "host": "localhost", - // }, - // { - // "name": "Python Attach 1", - // "type": "python", - // "request": "attach", - // "port": 3001, - // "host": "localhost" - // }, + { + "name": "Python Attach 0", + "type": "python", + "request": "attach", + "port": 3000, + "host": "localhost", + }, + { + "name": "Python Attach 1", + "type": "python", + "request": "attach", + "port": 3001, + "host": "localhost" + }, { "name": "Python Debugger: Current File", "type": "debugpy", diff --git a/spyro/io/basicio.py b/spyro/io/basicio.py index 1a35e83c..88bdc526 100644 --- a/spyro/io/basicio.py +++ b/spyro/io/basicio.py @@ -87,11 +87,11 @@ def wrapper(*args, **kwargs): u, u_r = func(*args, **dict(kwargs, source_nums=[snum])) return u, u_r elif args[0].parallelism_type == "custom": - shots_per_core_list = args[0].shots_per_core + shot_ids_per_propagation_list = args[0].shot_ids_per_propagation _comm = args[0].comm - for id_shots, shots_in_core in enumerate(shots_per_core_list): - if is_owner(_comm, id_shots): - u, u_r = func(*args, **dict(kwargs, source_nums=shots_in_core)) + for shot_ids_in_propagation in shot_ids_per_propagation_list: + if is_owner(_comm, shot_ids_in_propagation): + u, u_r = func(*args, **dict(kwargs, source_nums=shot_ids_in_propagation)) return u, u_r return wrapper @@ -313,7 +313,8 @@ def is_owner(ens_comm, rank): `True` if `rank` owns this shot """ - return ens_comm.ensemble_comm.rank == (rank % ens_comm.ensemble_comm.size) + owner = ens_comm.ensemble_comm.rank == (rank % ens_comm.ensemble_comm.size) + return owner def _check_units(c): diff --git a/spyro/io/model_parameters.py b/spyro/io/model_parameters.py index 8bdaa34d..62ee5220 100644 --- a/spyro/io/model_parameters.py +++ b/spyro/io/model_parameters.py @@ -1,4 +1,5 @@ import numpy as np +from mpi4py import MPI import warnings from .. import io from .. import utils @@ -542,11 +543,15 @@ def _sanitize_comm(self, comm): else: warnings.warn("No paralellism type listed. Assuming automatic") self.parallelism_type = "automatic" - + if self.parallelism_type == "custom": - self.shots_per_core = dictionary["parallelism"]["shots_per_core"] + self.shot_ids_per_propagation = dictionary["parallelism"]["shot_ids_per_propagation"] else: - self.shots_per_core = 1 + shot_ids_per_propagation = [] + available_cores = COMM_WORLD.size + num_cores_per_propagation = available_cores / self.number_of_sources + for shot in range(self.number_of_sources): + if comm is None: self.comm = utils.mpi_init(self) diff --git a/spyro/solvers/acoustic_wave.py b/spyro/solvers/acoustic_wave.py index fdc5cec1..5edb482f 100644 --- a/spyro/solvers/acoustic_wave.py +++ b/spyro/solvers/acoustic_wave.py @@ -44,6 +44,7 @@ def forward_solve(self): self.c = self.initial_velocity_model self.matrix_building() self.wave_propagator() + self.comm.comm.barrier() def force_rebuild_function_space(self): if self.mesh is None: diff --git a/spyro/utils/utils.py b/spyro/utils/utils.py index ebd105da..7cbb2d09 100644 --- a/spyro/utils/utils.py +++ b/spyro/utils/utils.py @@ -88,19 +88,19 @@ def mpi_init(model): available_cores = COMM_WORLD.size # noqa: F405 print(f"Parallelism type: {model.parallelism_type}", flush=True) if model.parallelism_type == "automatic": - num_cores_per_shot = available_cores / model.number_of_sources + num_cores_per_propagation = available_cores / model.number_of_sources if available_cores % model.number_of_sources != 0: raise ValueError( "Available cores cannot be divided between sources equally." ) elif model.parallelism_type == "spatial": - num_cores_per_shot = available_cores + num_cores_per_propagation = available_cores elif model.parallelism_type == "custom": - shots_per_core = model.shots_per_core - num_max_shots_per_core = max(len(sublist) for sublist in shots_per_core) - num_cores_per_shot = len(shots_per_core) + shot_ids_per_propagation = model.shot_ids_per_propagation + num_max_shots_per_core = max(len(sublist) for sublist in shot_ids_per_propagation) + num_cores_per_propagation = len(shot_ids_per_propagation) - comm_ens = Ensemble(COMM_WORLD, num_cores_per_shot) # noqa: F405 + comm_ens = Ensemble(COMM_WORLD, num_cores_per_propagation) # noqa: F405 return comm_ens diff --git a/temp_forward_shot.py b/temp_forward_shot.py index 325150ae..6fb27857 100644 --- a/temp_forward_shot.py +++ b/temp_forward_shot.py @@ -1,7 +1,7 @@ -# from mpi4py.MPI import COMM_WORLD -# import debugpy -# debugpy.listen(3000 + COMM_WORLD.rank) -# debugpy.wait_for_client() +from mpi4py.MPI import COMM_WORLD +import debugpy +debugpy.listen(3000 + COMM_WORLD.rank) +debugpy.wait_for_client() import spyro import numpy as np import math @@ -24,8 +24,7 @@ def run_forward(dt): # spyro however supports both spatial parallelism and "shot" parallelism. dictionary["parallelism"] = { "type": "custom", # options: automatic (same number of cores for evey processor) or spatial - "seperate_shots": True, - "shots_per_core": [[0, 1]], + "shot_ids_per_propagation": [[0, 1, 2]], } # Define the domain size without the PML. Here we'll assume a 1.00 x 1.00 km @@ -40,7 +39,7 @@ def run_forward(dt): } dictionary["acquisition"] = { "source_type": "ricker", - "source_locations": spyro.create_transect((-0.55, 0.7), (-0.55, 1.3), 2), + "source_locations": spyro.create_transect((-0.55, 0.7), (-0.55, 1.3), 3), "frequency": 5.0, "delay": 0.2, "delay_type": "time", diff --git a/temp_test_parallel.py b/temp_test_parallel.py new file mode 100644 index 00000000..01d8cbe1 --- /dev/null +++ b/temp_test_parallel.py @@ -0,0 +1,110 @@ +from mpi4py.MPI import COMM_WORLD +import debugpy +debugpy.listen(3000 + COMM_WORLD.rank) +debugpy.wait_for_client() +from mpi4py.MPI import COMM_WORLD +from mpi4py import MPI +import numpy as np +import firedrake as fire +import spyro +import warnings +warnings.filterwarnings("ignore") + + +def error_calc(p_numerical, p_analytical, nt): + norm = np.linalg.norm(p_numerical, 2) / np.sqrt(nt) + error_time = np.linalg.norm(p_analytical - p_numerical, 2) / np.sqrt(nt) + div_error_time = error_time / norm + return div_error_time + + +def test_forward_3_shots(): + final_time = 1.0 + + dictionary = {} + dictionary["options"] = { + "cell_type": "Q", # simplexes such as triangles or tetrahedra (T) or quadrilaterals (Q) + "variant": "lumped", # lumped, equispaced or DG, default is lumped + "degree": 4, # p order + "dimension": 2, # dimension + } + dictionary["parallelism"] = { + "type": "automatic", # options: automatic (same number of cores for evey processor) or spatial + } + dictionary["mesh"] = { + "Lz": 3.0, # depth in km - always positive # Como ver isso sem ler a malha? + "Lx": 3.0, # width in km - always positive + "Ly": 0.0, # thickness in km - always positive + "mesh_file": None, + "mesh_type": "firedrake_mesh", + } + dictionary["acquisition"] = { + "source_type": "ricker", + "source_locations": [(-1.1, 1.2), (-1.1, 1.5), (-1.1, 1.8)], + "frequency": 5.0, + "delay": 0.2, + "delay_type": "time", + "receiver_locations": spyro.create_transect((-1.3, 1.2), (-1.3, 1.8), 301), + } + dictionary["time_axis"] = { + "initial_time": 0.0, # Initial time for event + "final_time": final_time, # Final time for event + "dt": 0.001, # timestep size + "amplitude": 1, # the Ricker has an amplitude of 1. + "output_frequency": 100, # how frequently to output solution to pvds - Perguntar Daiane ''post_processing_frequnecy' + "gradient_sampling_frequency": 1, + } + dictionary["visualization"] = { + "forward_output": False, + "forward_output_filename": "results/forward_output.pvd", + "fwi_velocity_model_output": False, + "velocity_model_filename": None, + "gradient_output": False, + "gradient_filename": None, + } + + Wave_obj = spyro.AcousticWave(dictionary=dictionary) + Wave_obj.set_mesh(mesh_parameters={"dx": 0.1}) + + mesh_z = Wave_obj.mesh_z + cond = fire.conditional(mesh_z < -1.5, 3.5, 1.5) + Wave_obj.set_initial_velocity_model(conditional=cond, output=True) + + Wave_obj.forward_solve() + + comm = Wave_obj.comm + + arr = Wave_obj.receivers_output + + if comm.ensemble_comm.rank == 0: + analytical_p = spyro.utils.nodal_homogeneous_analytical( + Wave_obj, 0.2, 1.5, n_extra=100 + ) + else: + analytical_p = None + analytical_p = comm.ensemble_comm.bcast(analytical_p, root=0) + + # Checking if error before reflection matches + if comm.ensemble_comm.rank == 0: + rec_id = 0 + elif comm.ensemble_comm.rank == 1: + rec_id = 150 + elif comm.ensemble_comm.rank == 2: + rec_id = 300 + + arr0 = arr[:, rec_id] + arr0 = arr0.flatten() + + error = error_calc(arr0[:430], analytical_p[:430], 430) + if comm.comm.rank == 0: + print(f"Error for shot {Wave_obj.current_sources} is {error} and test has passed equals {np.abs(error) < 0.01}", flush=True) + error_all = COMM_WORLD.allreduce(error, op=MPI.SUM) + error_all /= 3 + + test = np.abs(error_all) < 0.01 + + assert test + + +if __name__ == "__main__": + test_forward_3_shots() diff --git a/velocity_models/tutorial b/velocity_models/tutorial deleted file mode 100644 index 81a389f542af76ce23026353c707775482c0aace..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 518568 zcmeI*e~g@UodEgBb^E~g@^ZhZ??`UJXv1!BPSYu|_%yVb&dhYD$|7+yAj5cN(8@4va zw~lTb+b}WKy3yEh|ASk`C#N=zHO3yA+OTEfHGdy{U}|D=e6%rs|AS+jHf$XoYpcL$ zW45v5C(Bo_{`QrPOV%|0d-)}+zWtpw%YWGT*B`A}ec2DMJgaTnPHv7ip6g_@j<*?Y zqYoJCM#=e@M9N z7blmds~U&6dJp<_43reNuumJ#fH{{qz`;Hq17QPOQpGsnoZGK+Zsc-Hid)#HnsN3M%88)ybRV<_f<_nAw_Hm3b zbAH$^Asxu(4ElT=D1Y$J`3Kv_9B5^5Ztkw1KRLj{J{|)c;K0Go6aF?F?F0^RK-PyN z>j&G<90=gRXFs{(VEf45FkfJ7ppRn%nK8q52{9%(e?0Qv|K$T^d++(R zDGqiFa3DZ?z`UXMfc8M86oqULaDW3G-~b0WzyS_$fCC)hfcXLraDW3G-~b0WpkKfN z4sd`29N+*4IKTl8aDW3G-~b1V6LNq99N+*4IKTn@0uFG1103K02ROh14sd`29N+*4 zIA9$U4sd`2<_kE$0S@RFaDW3G-~b0WzyS_$fCC)h00%h00qdA>fCC&bU%&wlaKJhy z9N+*4j1O>t103K02ROh14sd`29N+*4IKTn(1svc22ROh14sbxffCC)h00%h00S<70 z103K02ROh14j3oo00%h00S<701NsFV-~b0WzyS_$fCC)h00%h00S<7$Iwl<800+z$ zaDW3G&@bQs2ROh14sd`29N+*4IKTl8aDW5WG2s9QIAFej103Lhbxb(G0S*`+-~b0W zzyS_$fCC)h00%h00S<701Lg}jzyS_$fCC)hfPMi7IKTl8aDW3G-~b0WzyS_$fCC&b zPRIccaDW3G-~b2o3pl_54sd`29N+*4IKTl8aDW3G;DB{ZIKTl8m@nV}2RNW#zyS_$ zfCC)h00%h00S<70103K02drbl0S<7$d;teIzya%+aDW3GFh0Nm4sd`29N+*4IKTl8 zaDW3G-~b2A7jS?B9N+*4IKTn@0uFG1103K02ROh14sd`29N+*4IAENR103K02ROh1 z4(JzffCC)h00%h00S<70103K02ROh1>zHtW0~|14zyS_$K)-+k9N+*4IKTl8aDW3G z-~b0WzyS_e$AkkM;DGr84sd`2)-mA#2RLASfCC)h00%h00S<70103K02ROh14wx_C z00%h00S<701NsFV-~b0WzyS_$fCC)h00%h00S<7$I3Wi(zyS_$fCC)RFW>+NIKTl8 zaDW3G-~b0WzyS_$fCJVs;Q$9XV7`C@9N>U{0S7q10S<70103K02ROh14sd`29I%cF z2ROh1^93B>00*pN!T}C&!1w?MIKTl8aDW3G-~b0WzyS_$fCC&bU%&wlaDW3G-~b2o z3pl_54sd`29N+*4IKTl8aDW3G;DB*L4sd`29N+*4IG|s^0S<70103K02ROh14sd`2 z9N+*4tYg9f4sgJH0S7q10sR6FaDW3G-~b0WzyS_$fCC)h00%f=9TN_4fCJ_WIKTl8 zSjU6|9N>WQ0S<70103K02ROh14sd`29N+*4IAFej103K02ROh14(JzffCC)h00%h0 z0S<70103K02ROh1+NIKTl8aDW3G-~b0WzyS_$ zfCI(}Iluu9aDW3G;DCMs2ROh14sd`29N+*4IKTl8aDW3Gu#O1_IKTn(1svc22lNX# zzyS_$fCC)h00%h00S<70103Lhbxb(G0S=fi-~b0WU>y?00%h00S<6LzkmZA-~b0WzyS_$fCC)h00%h00S*`^U;OgO*+4wx_C00%grU%&wlaDW3G-~b0WzyS_$fCC)h00*pN!T}C&z0S<70103K02ROh14sd`29N+*4%olKg103K02ROh1{Q?effCC)h z00%h00S<70103K02RLAykOLgxz#uzt?H4ad51#wR=8_Y(HJ4m*NOnyxOdXwF=k7O0 zW{*r;&Kplx@4P+rIS<2Hex)G{P*}>^;1REbc>HgC>4o(8=I=F^eEF(ZGQZz~SKZGX zc=!+RNE@#BP`Y*Higd$|w-0L``}Mj5M6=Q=gfw7K#|NG{?Xg$oeIWzp+6XT8QTeYCk`>DKxA?&f}a{?K#Nttb6?nDW@K z*IH=^0~8k0fk9ixj%H2OeK@WA4l^GOw6+f8e(C>-&!BM?Qw7%mC4>v)?Im436s$XZNAq-GhNC)b^j>#=sPHirE-+g5{F!#La?!%TO5a?*AEU)kKD0$E~{n!Oi%sOzYR++tA3r8hA=>3AswjmIwl);t|~M8^Pf9S zEj>Nm@%7c|=gUW`&dMN{7a*FIRw1MVb)7f-{afChwqJJV{H&IJKkb~lfdj>k^Hdtb z0EKa&pyOxt1G>0$O1HPZyS(!Jj}P)Wy2{rUAexm{fdd6OkhRO7 z{>fzv<+SwQjU3lZe|giLDf=En!0+zWu}+nSFhF4(D9C{uTjPT}PkmSEnP2QP1&=QM zM!NQ_Q|p+yh;UKcXMQBML!%!FQ46SKO;1>B&#*-1qsyb>~i`wPPnmJipehP647>X%#{`P`CMl%z;0B7n&g<_ikgm;1+9Dh*+P!Z?ug*!ktocN>OGyL~83<`OCs~)LUpE|a~qt&_65C$lW z13Atit5;rRh-Rf#2F69p8?tXH*A-X4%B7d@Vd`_H9fles6}(xWt+aJ=Sh<%9l?Tp7I0vFjCIiK(67Dr zgGWZ1FPL*AZG6oCQI_-g;Q$9ZK9{aqHlC)w_SU)lMfsVgPqxMh2fCsxWWS&;pC9hVRz{VjRsL6ambICFH9LQdQ-xta`&yZR^4Qf8C zdR+q$?XF%zNC#@Nj!8rV{$8gyTwGON2DO~7LNovyhjgGO9Q zp!Z{DE_cy>a-jEpI@gZqDntXYaYzTu7qr$x$TtQ#cRc&qn+XFD?XF$|2YNeBvV7&u zYMGCh|K9g&_TCNKg#*1GFJzL%tThzyMAb?{` z&~XN)Po8WaqmOVqO?~aHY1Ojvj$eb%6Z$-13qA9OjRlW`j&ii|v5^SbZ?8jo>~oi; zYd?5oNA+jGqWfNZKzpF$3{9SNM0*A!+)j_KJ}O=J*{^o|8d$KP1sn+AKuzWg_GRs^ z(>FIqHoZ530a+c=#K$gTK#n%veiu2wfsS*u>Z0GIiSM2g#ejVd?D^fUj_a>2uwYvt zd;f;*VtgRSx#Q{`7iRtOC=R4u?>si$yZST^vFEzcIXnypAm4tid#<~!e@F-FG+(gsXX~3IC;lje1Fd?bR(%+d&w%Xx z8@3Av@*gwl`qOVn6L0%ybIC=`etM9#OZK%->N2(xxSkwXA7i{Q@Gz~N=qMr@wX1{WP1Kk`iY3DTuilb-y<&i**CbZa&~!GT60 z24wqx*e)T*3G1?sN&C6?_Nk-O%jZ1O7Y9aWR-}8D`TS7({>XWo0|D9tb(%NKj<;*i zIyF7I^c#IKAR8atdFs2;jXyZNt9DGzoD6zd90=e*P5wWkSF(`J7yRW-cQ!|k>&v`h zdip1qrJpY!d8Ixwp1Xg*0Swf&c{2aZm8OUwEeO>)9>H%?w;z*0S@d3FXT93-TDPto-zmS`1bA*AEVFE_vU5WqXhwxnpl?l~r^GJYQqj7>My1;_Uq2 zey!Vl!F&(yK5SXp24v2pTeh4kv&FXsviEe@F0aa8dlvh7yC&wW_V=AE1S@FAniVH zTl(#?mpZFMdFRVPFP{S(DC`)&dFzJs^u^yV%7HX<{f>0=vo{r1zuH?{o^u=s;6UBy z3)=Jb(~G~KCbxelzXc~RN%yb#1p~^eu54msOdeWcM^M{_B&jH^n&b=Sn zZSLSU2Ld=S2;+ov$J#BQx;Je<>L!0twhb23(yJ5F0rLeM z=yUt3^xy?6fCI({qMU;x)gxF;ORo+GI1nj+QR-59@PZY<0S-jTUZi>ii)rcA3F$!H z*D>jH-!@X7>ROl5gBPrTkPetHFizNKOM}I<^y-9kpziCK^!XUB>p3`5y-N>XumVCl zVBS!BpwE^Di)rcA;ec_%Nax`wbtye~!3y922cl#zQaysjwDjt5fCG{87o{$x2QOFw zAssMZV0^&%K&4V}Kzks{IXF^1Dh*+P!a_P=zCe3Gd!SMsIG{Zc={y{zE|rEbKw%u< zK$Pr7sz-omR$2uPa3E6tqSU3*5C$kLqyy#)j1L$es8k0IXb(g=2S=($r6CMZSV#xV z7ibS?4^*lH2ebzworj~;rP2@vD2xLfh?2cX^#~BnN~^#D4n)ddl)6+J!T^PZbig_$ z#s`cKRH{Qr2h1CCfCC&bU%&wlaDW3G-~b0WzyS_$fCC)h00*pN!T}C&z0S<70103K02ROh14sd`29N+*4%olKg103K02ROh1{Q?effCC)h00%h0 z0S<70103K02RLAykOLgx00%h00S@RFaDW3G-~b0WzyS_$fCC)h00%h00qdA>fCC&b zU%&wla6rF+103K02ROh14sd`29N+*4IKTl8SjU6|9N>WY0uFG11J*I&00%f=e1HQS z-~b0WzyS_$fCC)h00%h00S=fi-~b0WzyS_$fCKsk9N+*4IKTl8aDW3G-~b0WzyS_$ zz&IfXIKTl8aDW3G&@bQs2ROh14sd`29N+*4IKTl8aDW5WG2s9QIAFej103LhegOwK zzyS_$fCC)h00%h00S<70101l92?sd90rLeM-~b1#W5NLraKQKg2ROh14sd`29N+*4 wIKTl8aDW3GFkip{4sd`29N+*4^b0t^0S<70103K02ROh14sd`2eRkmg0mAntL;wH)