From c8688dbe5f9030fc10812ab27bdef95d9d2c616b Mon Sep 17 00:00:00 2001 From: Ritvik Rao Date: Mon, 14 Oct 2024 17:55:07 -0500 Subject: [PATCH 1/5] Check for pe num in pool examples and point to documentation --- examples/pool/pool_fibonacci.py | 3 +++ examples/pool/pool_simple.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/examples/pool/pool_fibonacci.py b/examples/pool/pool_fibonacci.py index f0153439..bbc81e47 100644 --- a/examples/pool/pool_fibonacci.py +++ b/examples/pool/pool_fibonacci.py @@ -9,6 +9,9 @@ def fib(n): return sum(charm.pool.map(fib, [n-1, n-2])) def main(args): + if(charm.numPes()==1): + print("Error: Run with more than one PE (exiting). For documentation on using pools, see: https://charm4py.readthedocs.io/en/latest/pool.html") + exit() print('fibonacci(13)=', fib(13)) exit() diff --git a/examples/pool/pool_simple.py b/examples/pool/pool_simple.py index 0adc7dd4..9dee0570 100644 --- a/examples/pool/pool_simple.py +++ b/examples/pool/pool_simple.py @@ -9,6 +9,9 @@ def twice(x): def main(args): ray.init() + if(charm.numPes()==1): + print("Error: Run with more than one PE (exiting). For documentation on using pools, see: https://charm4py.readthedocs.io/en/latest/pool.html") + exit() results = charm.pool.map_async(square, [4], chunksize=1, multi_future=True) results_twice = charm.pool.map_async(twice, results, chunksize=1, multi_future=True) From a2cdf2d0ae1a4a6d81fc9285c9a7b00d0438b76f Mon Sep 17 00:00:00 2001 From: Ritvik Rao Date: Mon, 14 Oct 2024 18:26:06 -0500 Subject: [PATCH 2/5] fix pool simple example --- examples/pool/pool_simple.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/pool/pool_simple.py b/examples/pool/pool_simple.py index 9dee0570..c5cde68a 100644 --- a/examples/pool/pool_simple.py +++ b/examples/pool/pool_simple.py @@ -13,7 +13,10 @@ def main(args): print("Error: Run with more than one PE (exiting). For documentation on using pools, see: https://charm4py.readthedocs.io/en/latest/pool.html") exit() results = charm.pool.map_async(square, [4], chunksize=1, multi_future=True) - results_twice = charm.pool.map_async(twice, results, chunksize=1, multi_future=True) + real_results = [] + for item in results: + real_results.append(item.get()) + results_twice = charm.pool.map_async(twice, real_results, chunksize=1, multi_future=True, is_ray=True) for x in results_twice: print(x.get()) From e7ac2004fdbf6b278d18f375f5d09353649213ea Mon Sep 17 00:00:00 2001 From: Ritvik Rao Date: Mon, 14 Oct 2024 18:30:10 -0500 Subject: [PATCH 3/5] change docs --- docs/pool.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/pool.rst b/docs/pool.rst index c37d9344..55632282 100644 --- a/docs/pool.rst +++ b/docs/pool.rst @@ -17,7 +17,8 @@ or creating more processes than processors. ``charm.pool`` is experimental, the API and performance (especially at large scales) is still subject to - change. + change. Additionally, do not include any Future objects in the iterable provided + to map or map_async, or to the args provided to Task. .. note:: From a2dbc06f6d65a9428ec1f0d2eefdcdae41fec8f3 Mon Sep 17 00:00:00 2001 From: Ritvik Rao Date: Tue, 15 Oct 2024 10:41:32 -0500 Subject: [PATCH 4/5] fix futures bug --- charm4py/charm.py | 4 ++-- charm4py/threads.py | 13 ++++++++++--- examples/pool/pool_simple.py | 7 ++----- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/charm4py/charm.py b/charm4py/charm.py index 47a9b1f4..e14e977f 100644 --- a/charm4py/charm.py +++ b/charm4py/charm.py @@ -1158,9 +1158,9 @@ def notify_future_deletion(self, store_id, depth): # if yes, remove it fut = charm.threadMgr.borrowed_futures[(store_id, depth)] refcount = ctypes.c_long.from_address(id(fut)).value - #print(store_id, "on pe", charm.myPe(), "depth", depth, "ref count =", refcount) + # print(store_id, "on pe", charm.myPe(), "depth", depth, "ref count =", refcount) if (fut.parent == None and refcount == 3) or (fut.parent != None and refcount == 2): - #print("Real deletion of", store_id, "from", charm.myPe()) + # print("Real deletion of", store_id, "from", charm.myPe()) if fut.parent == None: charm.threadMgr.futures.pop(fut.fid) charm.threadMgr.borrowed_futures.pop((store_id, depth)) diff --git a/charm4py/threads.py b/charm4py/threads.py index 3c6f0547..fc888e00 100644 --- a/charm4py/threads.py +++ b/charm4py/threads.py @@ -163,7 +163,13 @@ def __getstate__(self): # keep track of how many PEs this future is being sent to self.num_borrowers += 1 if self.store: - charm.threadMgr.borrowed_futures[(self.store_id, self.borrow_depth)] = self + # print("Getting state for id", self.store_id, "at borrow depth", self.borrow_depth, "num borrowers", self.num_borrowers, "on pe", charm.myPe()) + #in the case where one pe has 2 instances of futures with matching store ids and borrow depths + if (self.store_id, self.borrow_depth) in charm.threadMgr.borrowed_futures: + if charm.threadMgr.borrowed_futures[(self.store_id, self.borrow_depth)] is not self: + charm.threadMgr.borrowed_futures[(self.store_id, self.borrow_depth)].num_borrowers += 1 + else: + charm.threadMgr.borrowed_futures[(self.store_id, self.borrow_depth)] = self return (self.fid, self.src, self.store, self.borrow_depth, charm.myPe()) def __setstate__(self, state): @@ -173,6 +179,7 @@ def __setstate__(self, state): self.store_id = (self.src << 32) + self.fid else: self.store_id = 0 + # print("Setting state for id", self.store_id, "on pe", charm.myPe(), "at depth", self.borrow_depth) self._requested = False self.num_borrowers = 0 @@ -180,11 +187,11 @@ def __del__(self): if self.store: if self.parent == None and self.num_borrowers == 0: # This is the owner, delete the object from the object store - #print("Deleting owner", self.store_id) + # print("Deleting owner", self.store_id) self.delete_object() else: # this is a borrower, notify its parent of the deletion - #print("Deleting", self.store_id, "from", charm.myPe(), "sending notify to", self.parent) + # print("Deleting", self.store_id, "from", charm.myPe(), "sending notify to", self.parent, "at depth", self.borrow_depth) charm.thisProxy[self.parent].notify_future_deletion(self.store_id, self.borrow_depth - 1) diff --git a/examples/pool/pool_simple.py b/examples/pool/pool_simple.py index c5cde68a..ba546628 100644 --- a/examples/pool/pool_simple.py +++ b/examples/pool/pool_simple.py @@ -12,11 +12,8 @@ def main(args): if(charm.numPes()==1): print("Error: Run with more than one PE (exiting). For documentation on using pools, see: https://charm4py.readthedocs.io/en/latest/pool.html") exit() - results = charm.pool.map_async(square, [4], chunksize=1, multi_future=True) - real_results = [] - for item in results: - real_results.append(item.get()) - results_twice = charm.pool.map_async(twice, real_results, chunksize=1, multi_future=True, is_ray=True) + results = charm.pool.map_async(square, [4], chunksize=1, multi_future=True, is_ray=True) + results_twice = charm.pool.map_async(twice, results, chunksize=1, multi_future=True, is_ray=True) for x in results_twice: print(x.get()) From 8d3215e786b0ed75fb46d6cf1c2ac37fbd2f53b9 Mon Sep 17 00:00:00 2001 From: Ritvik Rao Date: Tue, 15 Oct 2024 14:20:49 -0500 Subject: [PATCH 5/5] small docs change --- docs/pool.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pool.rst b/docs/pool.rst index 55632282..d76e8185 100644 --- a/docs/pool.rst +++ b/docs/pool.rst @@ -18,7 +18,7 @@ or creating more processes than processors. ``charm.pool`` is experimental, the API and performance (especially at large scales) is still subject to change. Additionally, do not include any Future objects in the iterable provided - to map or map_async, or to the args provided to Task. + to map or map_async, unless you set is_ray to True. .. note::