[Carpet] [Cluster-users] job aborts with wierd error msg when Carpet does 1st recompose
Jonathan Thornburg
jthorn at aei.mpg.de
Wed Sep 27 17:06:53 CEST 2006
[[This thread started on the cluster-users at aei.mpg.de mailing list,
but the problem (Carpet dies the first time CarpetRegrid tries to
recompose the grid, with an assertion failure at line 1270 in data.cc)
now looks more like a generic Carpet problem, rather than a peyote
oddity.]]
Hi, Erik,
On Tue, 26 Sep 2006, Erik Schnetter wrote:
> I agree with Thomas that the first run is probably aborted by the same
> assertion failure.
>
> What is near line 1270 in your file data.cc? This depends on which version of
> Carpet you use; my version does not have an assert statement there.
Here is the relevant excerpt from Carpet/CarpetLib/src/data.cc,
run through 'cat -n':
1108 template<typename T>
1109 void data<T>
1110 ::interpolate_prolongate (const vector<const data<T>*> & srcs,
1111 const vector<CCTK_REAL> & times,
1112 const ibbox& box, const CCTK_REAL time,
1113 const int order_space,
1114 const int order_time)
1115 {
1116 const ibbox& sext = srcs[0]->extent();
1117 const ibbox& dext = extent();
1118
1119 int srcshp[dim], dstshp[dim];
1120 int srcbbox[dim][dim], dstbbox[dim][dim], regbbox[dim][dim];
1121
1122 fill_bbox_arrays (srcshp, dstshp, srcbbox, dstbbox, regbbox,
1123 box, sext, dext);
1124 switch (transport_operator) {
1125
[[...]]
1198 case op_Lagrange:
1199 switch (order_time) {
1200
1201 case 0:
1202 assert (times.size() == 1);
1203 assert (abs(times[0] - time) < eps);
1204 assert (srcs.size()>=1);
1205 switch (order_space) {
1206 case 1:
1207 if (all (sext.stride() == dext.stride() * 2)) {
1208 CCTK_FNAME(prolongate_3d_real8_rf2)
1209 ((const CCTK_REAL8*)srcs[0]->storage(),
1210 srcshp[0], srcshp[1], srcshp[2],
1211 (CCTK_REAL8*)storage(),
1212 dstshp[0], dstshp[1], dstshp[2],
1213 srcbbox, dstbbox, regbbox);
1214 } else {
1215 CCTK_FNAME(prolongate_3d_real8)
1216 ((const CCTK_REAL8*)srcs[0]->storage(),
1217 srcshp[0], srcshp[1], srcshp[2],
1218 (CCTK_REAL8*)storage(),
1219 dstshp[0], dstshp[1], dstshp[2],
1220 srcbbox, dstbbox, regbbox);
1221 }
1222 break;
1223 case 3:
1224 if (all (sext.stride() == dext.stride() * 2)) {
1225 CCTK_FNAME(prolongate_3d_real8_o3_rf2)
1226 ((const CCTK_REAL8*)srcs[0]->storage(),
1227 srcshp[0], srcshp[1], srcshp[2],
1228 (CCTK_REAL8*)storage(),
1229 dstshp[0], dstshp[1], dstshp[2],
1230 srcbbox, dstbbox, regbbox);
1231 } else {
1232 CCTK_FNAME(prolongate_3d_real8_o3)
1233 ((const CCTK_REAL8*)srcs[0]->storage(),
1234 srcshp[0], srcshp[1], srcshp[2],
1235 (CCTK_REAL8*)storage(),
1236 dstshp[0], dstshp[1], dstshp[2],
1237 srcbbox, dstbbox, regbbox);
1238 }
1239 break;
1240 case 5:
1241 if (all (sext.stride() == dext.stride() * 2)) {
1242 CCTK_FNAME(prolongate_3d_real8_o5_rf2)
1243 ((const CCTK_REAL8*)srcs[0]->storage(),
1244 srcshp[0], srcshp[1], srcshp[2],
1245 (CCTK_REAL8*)storage(),
1246 dstshp[0], dstshp[1], dstshp[2],
1247 srcbbox, dstbbox, regbbox);
1248 } else {
1249 CCTK_FNAME(prolongate_3d_real8_o5)
1250 ((const CCTK_REAL8*)srcs[0]->storage(),
1251 srcshp[0], srcshp[1], srcshp[2],
1252 (CCTK_REAL8*)storage(),
1253 dstshp[0], dstshp[1], dstshp[2],
1254 srcbbox, dstbbox, regbbox);
1255 }
1256 break;
1257 case 7:
1258 if (all (sext.stride() == dext.stride() * 2)) {
1259 CCTK_FNAME(prolongate_3d_real8_o7_rf2)
1260 ((const CCTK_REAL8*)srcs[0]->storage(),
1261 srcshp[0], srcshp[1], srcshp[2],
1262 (CCTK_REAL8*)storage(),
1263 dstshp[0], dstshp[1], dstshp[2],
1264 srcbbox, dstbbox, regbbox);
1265 } else {
1266 assert (0);
1267 }
1268 break;
1269 default:
1270 assert (0);
1271 }
1272 break;
That is, the assertion failure is saying that order_space had an
unexpected value. My par file set
Carpet::domain_from_coordbase = true
Carpet::use_outer_buffer_zones = true
Driver::ghost_size = 3
Carpet::num_integrator_substeps = 4
Carpet::prolongation_order_space = 4
Carpet::prolongation_order_time = 2
and
ADMMacros::spatial_order = 4
I will retry my run with a darcs-stable3 or darcs-development pull,
and also see if I can replicate this on a single processor.
In the mean time, do you see anything wrong with my full par file
(attached)?
thanks, ciao,
--
-- Jonathan Thornburg <jthorn at aei.mpg.de>
Max-Planck-Institut fuer Gravitationsphysik (Albert-Einstein-Institut),
Golm, Germany, "Old Europe" http://www.aei.mpg.de/~jthorn/home.html
"Washing one's hands of the conflict between the powerful and the
powerless means to side with the powerful, not to be neutral."
-- quote by Freire / poster by Oxfam
-------------- next part --------------
Cactus::cctk_run_title = "Schwarzschild octant generic-RK4 Lego-excision dx=0.100"
Cactus::cctk_full_warnings = yes
########################################
#
# time integration
#
Cactus::terminate = "time"
Cactus::cctk_final_time = 100.0
ActiveThorns = "Time"
Time::dtfac = 0.666666666666666
ActiveThorns = "MoL"
MoL::ODE_Method = "generic"
MoL::MoL_Intermediate_Steps = 4
MoL::MoL_Num_Scratch_Levels = 3
MoL::initial_data_is_crap = true
MoL::copy_ID_after_MoL_PostStep = true
########################################
#
# general utilities
#
ActiveThorns = "IOUtil"
IOUtil::parfile_write = "no"
IO::out_dir = $parfile # n.b. must *not* use "quotes" here!!!
ActiveThorns = "Fortran"
########################################
#
# grid/Carpet
#
ActiveThorns = "Boundary"
ActiveThorns = "SymBase"
ActiveThorns = "CarpetLib Carpet"
Carpet::silent = true # supress points/second etc messages
Carpet::domain_from_coordbase = true
Carpet::use_outer_buffer_zones = true
Driver::ghost_size = 3
Carpet::num_integrator_substeps = 4
Carpet::prolongation_order_space = 4
Carpet::prolongation_order_time = 2
ActiveThorns = "CoordBase"
CoordBase::domainsize = "minmax"
CoordBase::xmin = 0.0
CoordBase::ymin = 0.0
CoordBase::zmin = 0.0
CoordBase::xmax = 64.0
CoordBase::ymax = 64.0
CoordBase::zmax = 64.0
CoordBase::dx = 1.6
CoordBase::dy = 1.6
CoordBase::dz = 1.6
CoordBase::boundary_size_x_lower = 3
CoordBase::boundary_size_y_lower = 3
CoordBase::boundary_size_z_lower = 3
CoordBase::boundary_shiftout_x_lower = 1
CoordBase::boundary_shiftout_y_lower = 1
CoordBase::boundary_shiftout_z_lower = 1
ActiveThorns = "CartGrid3D"
Grid::domain = "octant"
CartGrid3D::type = "coordbase"
CartGrid3D::avoid_origin = false
ActiveThorns = "CarpetRegrid"
Carpet::max_refinement_levels = 5
CarpetRegrid::refinement_levels = 5
CarpetRegrid::refined_regions = "manual-coordinate-list"
CarpetRegrid::coordinates = "[
[ ([0.0,0.0,0.0]:[32.0,32.0,32.0]:[0.800,0.800,0.800])],
[ ([0.0,0.0,0.0]:[16.0,16.0,16.0]:[0.400,0.400,0.400])],
[ ([0.0,0.0,0.0]:[ 8.0, 8.0, 8.0]:[0.200,0.200,0.200])],
[ ([0.0,0.0,0.0]:[ 4.0, 4.0, 4.0]:[0.100,0.100,0.100])]
]"
CarpetRegrid::smart_outer_boundaries = true
CarpetRegrid::keep_same_grid_structure = true
CarpetLib::save_memory_during_regridding = true
########################################
#
# slabbing & interpolation
#
ActiveThorns = "Slab CarpetSlab"
ActiveThorns = "LocalReduce CarpetReduce"
ActiveThorns = "AEILocalInterp CarpetInterp"
########################################
#
# .not. NaNs
#
ActiveThorns = "NaNChecker"
NaNChecker::action_if_found = "terminate"
NaNChecker::check_every = 64 # 4 coarse time steps
NaNChecker::check_after = 64 # 4 coarse time steps
NaNChecker::report_max = 1
NaNChecker::check_vars = "
ADM_BSSN::ADM_BSSN_metric
"
########################################
#
# CactusEinstein & BSSN_MoL evolution
#
ActiveThorns = "ADMBase StaticConformal ADMCoupling CoordGauge"
ActiveThorns = "ADMMacros"
ADMMacros::spatial_order = 4
ActiveThorns = "SpaceMask"
ActiveThorns = "ADMConstraints"
ADMBase::metric_type = "physical"
ActiveThorns = "BSSN_MoL"
ADMBase::evolution_method = "adm_bssn"
BSSN_MoL::advection = "upwind4"
BSSN_MoL::ICN_itnum = 0 # we're not using ICN!
BSSN_MoL::stencil_size = 3
########################################
#
# gauge
#
ADMBase::lapse_evolution_method = "1+log"
ADMBase::initial_lapse = "one"
BSSN_MoL::lapsesource = "modified"
BSSN_MoL::harmonic_f = 2.0
ADMBase::initial_shift = "zero"
ADMBase::shift_evolution_method = "gamma0"
BSSN_MoL::ShiftGammaCoeff = 0.75
BSSN_MoL::BetaDriver = 4
BSSN_MoL::ShiftAlpPower = 0
BSSN_MoL::ShiftPsiPower = 0
########################################
#
# LEGO excision
#
SpaceMask::use_mask = true
ActiveThorns = "LegoExcision"
BSSN_MoL::excise = true
BSSN_MoL::excisiontype = "lego"
ADMConstraints::excise = true
########################################
#
# outer BCs
#
BSSN_MoL::bound = "newrad"
Boundary::radpower = 3
########################################
#
# Initial data
#
ActiveThorns = "Kerr"
Kerr::mass = 1.0
Kerr::spin = 0.0
########################################
#
# keep track of spherical surfaces for later use
#
ActiveThorns = "SphericalSurface"
SphericalSurface::nsurfaces = 1
SphericalSurface::maxntheta = 39
SphericalSurface::maxnphi = 76
SphericalSurface::ntheta [0] = 22
SphericalSurface::nphi [0] = 76
SphericalSurface::nghoststheta [0] = 2
SphericalSurface::nghostsphi [0] = 2
########################################
#
# find AHs & set excision mask
#
ActiveThorns = "AHFinderDirect"
AHFinderDirect::N_horizons = 1
AHFinderDirect::find_every = 15 # every t=1
AHFinderDirect::max_Newton_iterations__initial = 30
AHFinderDirect::max_Newton_iterations__subsequent = 30
AHFinderDirect::verbose_level = "physics details"
AHFinderDirect::output_h_every = 150 # every t=10
AHFinderDirect::patch_system_type [1] = "+xyz octant (mirrored)"
AHFinderDirect::initial_guess_method [1] = Kerr/Kerr-Schild
AHFinderDirect::initial_guess__Kerr_KerrSchild__mass[1] = 1.0
AHFinderDirect::initial_guess__Kerr_KerrSchild__spin[1] = 0.0
AHFinderDirect::which_surface_to_store_info [1] = 0
AHFinderDirect::max_allowable_horizon_radius [1] = 6.0
AHFinderDirect::set_mask_for_all_horizons = true
AHFinderDirect::set_old_style_mask = true
AHFinderDirect::mask_radius_multiplier = 0.6
AHFinderDirect::mask_radius_offset = 0.0
AHFinderDirect::mask_buffer_thickness = 0.0
AHFinderDirect::old_style_mask_buffer_value = 0.0
########################################
#
# output
#
ActiveThorns = "CarpetIOBasic"
CarpetIOBasic::outInfo_every = 16 # every coarse time step
CarpetIOBasic::outInfo_reductions = "norm_inf norm2"
CarpetIOBasic::outInfo_vars = "
ADMConstraints::hamiltonian
ADM_BSSN::ADM_BSSN_K
"
ActiveThorns = "CarpetIOASCII"
IOASCII::out1D_criterion = "time"
IOASCII::out1D_dt = 1.0
IOASCII::out1D_vars = "
ADMBase::lapse
ADMBase::shift
ADM_BSSN::ADM_BSSN_K
ADMConstraints::hamiltonian
SpaceMask::emask
"
ActiveThorns = "CarpetIOHDF5"
IOHDF5::out_criterion = "time"
IOHDF5::out_dt = 1.0
IOHDF5::out_vars = "
ADMConstraints::hamiltonian
"
More information about the developers
mailing list