thread 'tokio-runtime-worker' panicked at fendermint/vm/topdown/src/finality/null.rs:111:9:
assertion failed: previous_finality == self.last_committed_finality.read_clone()?
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
2024-11-20T13:53:09.593342Z ERROR fendermint/app/src/main.rs:24: panicking stacktrace="   0: std::backtrace_rs::backtrace::libunwind::trace
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/../../backtrace/src/backtrace/libunwind.rs:116:5
   1: std::backtrace_rs::backtrace::trace_unsynchronized
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2: std::backtrace::Backtrace::create
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/backtrace.rs:331:13
   3: fendermint::init_panic_handler::{{closure}}
             at /Users/alexei/projects/ipc/fendermint/app/src/main.rs:21:26
   4: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2084:9
   5: std::panicking::rust_panic_with_hook
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:808:13
   6: std::panicking::begin_panic_handler::{{closure}}
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:667:13
   7: std::sys::backtrace::__rust_end_short_backtrace
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:168:18
   8: rust_begin_unwind
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:665:5
   9: core::panicking::panic_fmt
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panicking.rs:74:14
  10: core::panicking::panic
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panicking.rs:148:5
  11: fendermint_vm_topdown::finality::null::FinalityWithNull::set_new_finality
             at /Users/alexei/projects/ipc/fendermint/vm/topdown/src/finality/null.rs:111:9
  12: <fendermint_vm_topdown::finality::fetch::CachedFinalityProvider<T> as fendermint_vm_topdown::ParentFinalityProvider>::set_new_finality
             at /Users/alexei/projects/ipc/fendermint/vm/topdown/src/finality/fetch.rs:127:9
  13:<fendermint_vm_topdown::toggle::Toggle<P> as fendermint_vm_topdown::ParentFinalityProvider>::set_new_finality::{{closure}}
             at /Users/alexei/projects/ipc/fendermint/vm/topdown/src/toggle.rs:90:34
  14: fendermint_vm_topdown::toggle::Toggle<P>::perform_or_else
             at /Users/alexei/projects/ipc/fendermint/vm/topdown/src/toggle.rs:38:24
  15: <fendermint_vm_topdown::toggle::Toggle<P> as fendermint_vm_topdown::ParentFinalityProvider>::set_new_finality
             at /Users/alexei/projects/ipc/fendermint/vm/topdown/src/toggle.rs:90:9
  16: <fendermint_vm_interpreter::chain::ChainMessageInterpreter<I,DB> as fendermint_vm_interpreter::ExecInterpreter>::deliver::{{closure}}::{{closure}}
             at /Users/alexei/projects/ipc/fendermint/vm/interpreter/src/chain.rs:615:25
  17: async_stm::ops::atomically::{{closure}}::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-stm-0.4.0/src/ops.rs:85:34
  18: async_stm::ops::atomically_aux::{{closure}}::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-stm-0.4.0/src/ops.rs:96:56
  19: async_stm::ops::atomically_or_err_aux::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-stm-0.4.0/src/ops.rs:144:22
  20: async_stm::ops::atomically_aux::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-stm-0.4.0/src/ops.rs:97:10
  21: async_stm::ops::atomically::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-stm-0.4.0/src/ops.rs:85:39
  22: <fendermint_vm_interpreter::chain::ChainMessageInterpreter<I,DB> as fendermint_vm_interpreter::ExecInterpreter>::deliver::{{closure}}
             at /Users/alexei/projects/ipc/fendermint/vm/interpreter/src/chain.rs:627:22
  23: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
  24: <fendermint_vm_interpreter::bytes::BytesMessageInterpreter<I> as fendermint_vm_interpreter::ExecInterpreter>::deliver::{{closure}}
             at /Users/alexei/projects/ipc/fendermint/vm/interpreter/src/bytes.rs:198:67
  25: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
  26: <fendermint_app::app::App<DB,SS,S,I,C> as fendermint_abci::application::Application>::deliver_tx::{{closure}}::{{closure}}::{{closure}}
             at /Users/alexei/projects/ipc/fendermint/app/src/app.rs:766:76
  27: fendermint_app::app::App<DB,SS,S,I,C>::modify_exec_state::{{closure}}
             at /Users/alexei/projects/ipc/fendermint/app/src/app.rs:323:71
  28: <fendermint_app::app::App<DB,SS,S,I,C> as fendermint_abci::application::Application>::deliver_tx::{{closure}}
             at /Users/alexei/projects/ipc/fendermint/app/src/app.rs:770:14
  29: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
  30: <fendermint_abci::application::ApplicationService<A> as tower_service::Service<tendermint::v0_37::abci::request::Request>>::call::{{closure}}
             at /Users/alexei/projects/ipc/fendermint/abci/src/application.rs:180:90
  31: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
  32: <tower_abci::buffer4::future::ResponseFuture<F> as core::future::future::Future>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tower-abci-0.7.0/src/buffer4/future.rs:60:56
  33: <tower_abci::split::futures::ConsensusFuture<S> as core::future::future::Future>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tower-abci-0.7.0/src/split.rs:249:19
  34: <tower::limit::concurrency::future::ResponseFuture<F> as core::future::future::Future>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tower-0.4.13/src/limit/concurrency/future.rs:39:28
  35: <tower::buffer::future::ResponseFuture<F> as core::future::future::Future>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tower-0.4.13/src/buffer/future.rs:75:59
  36: <F as futures_core::future::TryFuture>::try_poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.31/src/future.rs:92:9
  37: <futures_util::future::try_future::into_future::IntoFuture<Fut> as core::future::future::Future>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/try_future/into_future.rs:34:9
  38: <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/future/map.rs:55:37
  39: <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/lib.rs:86:13
  40: <futures_util::future::try_future::MapOk<Fut,F> as core::future::future::Future>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/lib.rs:86:13
  41: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
  42: <futures_util::stream::futures_ordered::OrderWrapper<T> as core::future::future::Future>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/stream/futures_ordered.rs:56:9
  43: <futures_util::stream::futures_unordered::FuturesUnordered<Fut> as futures_core::stream::Stream>::poll_next
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/stream/futures_unordered/mod.rs:528:17
  44: futures_util::stream::stream::StreamExt::poll_next_unpin
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/stream/stream/mod.rs:1638:9
  45: <futures_util::stream::futures_ordered::FuturesOrdered<Fut> as futures_core::stream::Stream>::poll_next
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/stream/futures_ordered.rs:195:26
  46: futures_util::stream::stream::StreamExt::poll_next_unpin
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/stream/stream/mod.rs:1638:9
  47: <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/stream/stream/next.rs:32:9
  48: tower_abci::v037::server::Connection<C,M,I,S>::run::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tower-abci-0.7.0/src/v037/server.rs:226:73
  49: tower_abci::v037::server::Server<C,M,I,S>::listen::{{closure}}::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tower-abci-0.7.0/src/v037/server.rs:145:64
  50: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
  51: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/core.rs:331:17
  52: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/loom/std/unsafe_cell.rs:16:9
  53: tokio::runtime::task::core::Core<T,S>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/core.rs:320:13
  54: tokio::runtime::task::harness::poll_future::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/harness.rs:499:19
  55: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
  56: std::panicking::try::do_call
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
  57: ___rust_try
  58: std::panicking::try
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
  59: std::panic::catch_unwind
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
  60: tokio::runtime::task::harness::poll_future
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/harness.rs:487:18
  61: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/harness.rs:209:27
  62: tokio::runtime::task::harness::Harness<T,S>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/harness.rs:154:15
  63: tokio::runtime::task::raw::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/raw.rs:271:5
  64: tokio::runtime::task::raw::RawTask::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/raw.rs:201:18
  65: tokio::runtime::task::LocalNotified<S>::run
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/mod.rs:435:9
  66: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/scheduler/multi_thread/worker.rs:596:13
  67: tokio::runtime::coop::with_budget
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/coop.rs:107:5
  68: tokio::runtime::coop::budget
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/coop.rs:73:5
  69: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
 at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/scheduler/multi_thread/worker.rs:595:9
  70: tokio::runtime::scheduler::multi_thread::worker::Context::run
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/scheduler/multi_thread/worker.rs:546:24
  71: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/scheduler/multi_thread/worker.rs:511:21
  72: tokio::runtime::context::scoped::Scoped<T>::set
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/context/scoped.rs:40:9
  73: tokio::runtime::context::set_scheduler::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/context.rs:180:26
  74: std::thread::local::LocalKey<T>::try_with
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:283:12
  75: std::thread::local::LocalKey<T>::with
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:260:9
  76: tokio::runtime::context::set_scheduler
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/context.rs:180:9
  77: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/scheduler/multi_thread/worker.rs:506:9
  78: tokio::runtime::context::runtime::enter_runtime
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/context/runtime.rs:65:16
  79: tokio::runtime::scheduler::multi_thread::worker::run
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/scheduler/multi_thread/worker.rs:498:5
  80: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/scheduler/multi_thread/worker.rs:464:45
  81: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/blocking/task.rs:42:21
  82: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/core.rs:331:17
  83: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/loom/std/unsafe_cell.rs:16:9
  84: tokio::runtime::task::core::Core<T,S>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/core.rs:320:13
  85: tokio::runtime::task::harness::poll_future::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/harness.rs:499:19
  86: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
  87: std::panicking::try::do_call
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
  88: ___rust_try
  89: std::panicking::try
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
  90: std::panic::catch_unwind
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
  91: tokio::runtime::task::harness::poll_future
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/harness.rs:487:18
  92: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/harness.rs:209:27
  93: tokio::runtime::task::harness::Harness<T,S>::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/harness.rs:154:15
  94: tokio::runtime::task::raw::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/raw.rs:271:5
  95: tokio::runtime::task::raw::RawTask::poll
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/raw.rs:201:18
  96: tokio::runtime::task::UnownedTask<S>::run
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/task/mod.rs:472:9
  97: tokio::runtime::blocking::pool::Task::run
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/blocking/pool.rs:161:9
  98: tokio::runtime::blocking::pool::Inner::run
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/blocking/pool.rs:511:17
  99: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}
             at /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.0/src/runtime/blocking/pool.rs:469:13
 100: std::sys::backtrace::__rust_begin_short_backtrace
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:152:18
 101: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:538:17
 102: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
 103: std::panicking::try::do_call
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
 104: ___rust_try
 105: std::panicking::try
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
 106: std::panic::catch_unwind
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
 107: std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:537:30
 108: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5
 109: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9
 110: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9
 111: std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/pal/unix/thread.rs:108:17
 112: __pthread_joiner_wake
" info="panicked at fendermint/vm/topdown/src/finality/null.rs:111:9:
assertion failed: previous_finality == self.last_committed_finality.read_clone()?"

ipc-cli checkpoint relayer --subnet $subnet_id --checkpoint-interval-sec 15 --submitter $submitter_addr --max-parallelism 1

2024-11-20T15:41:49.824304Z  INFO ipc_provider::checkpoint: last submission height: 60
2024-11-20T15:41:50.863185Z  INFO ipc_provider::checkpoint: submitted bottom up checkpoint(70) in parent at height 43
2024-11-20T15:41:50.863231Z  INFO tracing_event: domain="Bottomup" event=CheckpointSubmitted { height: 70, hash: 63073049d529d4adc79f09921031033ebce1f2a9cf64dca47f3a0e4bd9793dce }
2024-11-20T15:41:51.874074Z  INFO ipc_provider::checkpoint: submitted bottom up checkpoint(80) in parent at height 44
2024-11-20T15:41:51.874117Z  INFO tracing_event: domain="Bottomup" event=CheckpointSubmitted { height: 80, hash: b29bbe8a6d336633b6f8405be98eb667bad77f68229b99ce834b3382dcef1bfb }
2024-11-20T15:42:06.876987Z  INFO ipc_provider::checkpoint: last submission height: 80
2024-11-20T15:42:07.918524Z  INFO ipc_provider::checkpoint: submitted bottom up checkpoint(90) in parent at height 45
2024-11-20T15:42:07.918549Z  INFO tracing_event: domain="Bottomup" event=CheckpointSubmitted { height: 90, hash: 012efc65ea01f91de45a3a339e7f31a8bcef296723a5626946991e14255feb1c }
2024-11-20T15:42:22.922111Z  INFO ipc_provider::checkpoint: last submission height: 90
2024-11-20T15:42:23.964964Z  INFO ipc_provider::checkpoint: submitted bottom up checkpoint(100) in parent at height 46
2024-11-20T15:42:23.965011Z  INFO tracing_event: domain="Bottomup" event=CheckpointSubmitted { height: 100, hash: bea022c8e1fe64c77ffbb9d5133975ba744a0fa9db252c7013bbb1f452ea90a6 }
2024-11-20T15:42:24.975752Z  INFO ipc_provider::checkpoint: submitted bottom up checkpoint(110) in parent at height 47
2024-11-20T15:42:24.975800Z  INFO tracing_event: domain="Bottomup" event=CheckpointSubmitted { height: 110, hash: 897d4c94b7c4c1c3e0fbee85db47fe93ae905361f641809e49dccd668a3123a4 }
2024-11-20T15:42:39.984029Z  INFO ipc_provider::checkpoint: last submission height: 110
2024-11-20T15:42:39.987508Z ERROR ipc_provider::checkpoint: cannot submit checkpoint for submitter: t410ftfsva7i2kw6me2k4lc5bn6zx3am3bjg466vg7ji due to (code: 0, message: Tendermint RPC error: HTTP error

Caused by:
    error trying to connect: tcp connect error: Connection refused (os error 61)

Location:
    /Users/alexei/.cargo/registry/src/index.crates.io-6f17d22bba15001f/flex-error-0.4.4/src/tracer_impl/eyre.rs:10:9, data: None)