From 9f81f828c797f842d1df0a5cbda898bc0df8075a Mon Sep 17 00:00:00 2001 From: mingyu Date: Mon, 24 Feb 2025 05:07:33 +0900 Subject: [PATCH] gh-129948: Add `set()` to `multiprocessing.managers.SyncManager` (#129949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SyncManager provided support for various data structures such as dict, list, and queue, but oddly, not set. This introduces support for set by defining SetProxy and registering it with SyncManager. --- Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Gregory P. Smith --- Doc/library/multiprocessing.rst | 20 ++- Doc/whatsnew/3.14.rst | 5 + Lib/multiprocessing/managers.py | 31 ++++ Lib/test/_test_multiprocessing.py | 144 ++++++++++++++++++ ...-02-11-06-42-17.gh-issue-129948.ZcugY9.rst | 2 + 5 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-02-11-06-42-17.gh-issue-129948.ZcugY9.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 1b1c6d74e51..41ade9f2d94 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -380,35 +380,40 @@ However, if you really do need to use some shared data then proxies. A manager returned by :func:`Manager` will support types - :class:`list`, :class:`dict`, :class:`~managers.Namespace`, :class:`Lock`, + :class:`list`, :class:`dict`, :class:`set`, :class:`~managers.Namespace`, :class:`Lock`, :class:`RLock`, :class:`Semaphore`, :class:`BoundedSemaphore`, :class:`Condition`, :class:`Event`, :class:`Barrier`, :class:`Queue`, :class:`Value` and :class:`Array`. For example, :: from multiprocessing import Process, Manager - def f(d, l): + def f(d, l, s): d[1] = '1' d['2'] = 2 d[0.25] = None l.reverse() + s.add('a') + s.add('b') if __name__ == '__main__': with Manager() as manager: d = manager.dict() l = manager.list(range(10)) + s = manager.set() - p = Process(target=f, args=(d, l)) + p = Process(target=f, args=(d, l, s)) p.start() p.join() print(d) print(l) + print(s) will print :: {0.25: None, 1: '1', '2': 2} [9, 8, 7, 6, 5, 4, 3, 2, 1, 0] + {'a', 'b'} Server process managers are more flexible than using shared memory objects because they can be made to support arbitrary object types. Also, a single @@ -1942,6 +1947,15 @@ their parent process exits. The manager classes are defined in the Create a shared :class:`list` object and return a proxy for it. + .. method:: set() + set(sequence) + set(mapping) + + Create a shared :class:`set` object and return a proxy for it. + + .. versionadded:: next + :class:`set` support was added. + .. versionchanged:: 3.6 Shared objects are capable of being nested. For example, a shared container object such as a shared list can contain other shared objects diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 930fe718ac5..1cd8da46a2b 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -700,6 +700,11 @@ multiprocessing (Contributed by Roy Hyunjin Han for :gh:`103134`.) +* Add support for shared :class:`set` objects via + :meth:`SyncManager.set() `. + The :func:`set` in :func:`multiprocessing.Manager` method is now available. + (Contributed by Mingyu Park in :gh:`129949`.) + operator -------- diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 040f4674d73..c1f09d2b409 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1195,6 +1195,36 @@ def __ior__(self, value): collections.abc.MutableMapping.register(_BaseDictProxy) +_BaseSetProxy = MakeProxyType("_BaseSetProxy", ( + '__and__', '__class_getitem__', '__contains__', '__iand__', '__ior__', + '__isub__', '__iter__', '__ixor__', '__len__', '__or__', '__rand__', + '__ror__', '__rsub__', '__rxor__', '__sub__', '__xor__', + '__ge__', '__gt__', '__le__', '__lt__', + 'add', 'clear', 'copy', 'difference', 'difference_update', 'discard', + 'intersection', 'intersection_update', 'isdisjoint', 'issubset', + 'issuperset', 'pop', 'remove', 'symmetric_difference', + 'symmetric_difference_update', 'union', 'update', +)) + +class SetProxy(_BaseSetProxy): + def __ior__(self, value): + self._callmethod('__ior__', (value,)) + return self + def __iand__(self, value): + self._callmethod('__iand__', (value,)) + return self + def __ixor__(self, value): + self._callmethod('__ixor__', (value,)) + return self + def __isub__(self, value): + self._callmethod('__isub__', (value,)) + return self + + __class_getitem__ = classmethod(types.GenericAlias) + +collections.abc.MutableMapping.register(_BaseSetProxy) + + ArrayProxy = MakeProxyType('ArrayProxy', ( '__len__', '__getitem__', '__setitem__' )) @@ -1245,6 +1275,7 @@ class SyncManager(BaseManager): SyncManager.register('Pool', pool.Pool, PoolProxy) SyncManager.register('list', list, ListProxy) SyncManager.register('dict', dict, DictProxy) +SyncManager.register('set', set, SetProxy) SyncManager.register('Value', Value, ValueProxy) SyncManager.register('Array', Array, ArrayProxy) SyncManager.register('Namespace', Namespace, NamespaceProxy) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4b7c3e7fa8b..5dd89bd5af7 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6441,6 +6441,150 @@ def test_namespace(self): o.y = 1 self.run_worker(self._test_namespace, o) + @classmethod + def _test_set_operator_symbols(cls, obj): + case = unittest.TestCase() + obj.update(['a', 'b', 'c']) + case.assertEqual(len(obj), 3) + case.assertIn('a', obj) + case.assertNotIn('d', obj) + result = obj | {'d', 'e'} + case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'}) + result = {'d', 'e'} | obj + case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'}) + obj |= {'d', 'e'} + case.assertSetEqual(obj, {'a', 'b', 'c', 'd', 'e'}) + case.assertIsInstance(obj, multiprocessing.managers.SetProxy) + + obj.clear() + obj.update(['a', 'b', 'c']) + result = {'a', 'b', 'd'} - obj + case.assertSetEqual(result, {'d'}) + result = obj - {'a', 'b'} + case.assertSetEqual(result, {'c'}) + obj -= {'a', 'b'} + case.assertSetEqual(obj, {'c'}) + case.assertIsInstance(obj, multiprocessing.managers.SetProxy) + + obj.clear() + obj.update(['a', 'b', 'c']) + result = {'b', 'c', 'd'} ^ obj + case.assertSetEqual(result, {'a', 'd'}) + result = obj ^ {'b', 'c', 'd'} + case.assertSetEqual(result, {'a', 'd'}) + obj ^= {'b', 'c', 'd'} + case.assertSetEqual(obj, {'a', 'd'}) + case.assertIsInstance(obj, multiprocessing.managers.SetProxy) + + obj.clear() + obj.update(['a', 'b', 'c']) + result = obj & {'b', 'c', 'd'} + case.assertSetEqual(result, {'b', 'c'}) + result = {'b', 'c', 'd'} & obj + case.assertSetEqual(result, {'b', 'c'}) + obj &= {'b', 'c', 'd'} + case.assertSetEqual(obj, {'b', 'c'}) + case.assertIsInstance(obj, multiprocessing.managers.SetProxy) + + obj.clear() + obj.update(['a', 'b', 'c']) + case.assertSetEqual(set(obj), {'a', 'b', 'c'}) + + @classmethod + def _test_set_operator_methods(cls, obj): + case = unittest.TestCase() + obj.add('d') + case.assertIn('d', obj) + + obj.clear() + obj.update(['a', 'b', 'c']) + copy_obj = obj.copy() + case.assertSetEqual(copy_obj, obj) + obj.remove('a') + case.assertNotIn('a', obj) + case.assertRaises(KeyError, obj.remove, 'a') + + obj.clear() + obj.update(['a']) + obj.discard('a') + case.assertNotIn('a', obj) + obj.discard('a') + case.assertNotIn('a', obj) + obj.update(['a']) + popped = obj.pop() + case.assertNotIn(popped, obj) + + obj.clear() + obj.update(['a', 'b', 'c']) + result = obj.intersection({'b', 'c', 'd'}) + case.assertSetEqual(result, {'b', 'c'}) + obj.intersection_update({'b', 'c', 'd'}) + case.assertSetEqual(obj, {'b', 'c'}) + + obj.clear() + obj.update(['a', 'b', 'c']) + result = obj.difference({'a', 'b'}) + case.assertSetEqual(result, {'c'}) + obj.difference_update({'a', 'b'}) + case.assertSetEqual(obj, {'c'}) + + obj.clear() + obj.update(['a', 'b', 'c']) + result = obj.symmetric_difference({'b', 'c', 'd'}) + case.assertSetEqual(result, {'a', 'd'}) + obj.symmetric_difference_update({'b', 'c', 'd'}) + case.assertSetEqual(obj, {'a', 'd'}) + + @classmethod + def _test_set_comparisons(cls, obj): + case = unittest.TestCase() + obj.update(['a', 'b', 'c']) + result = obj.union({'d', 'e'}) + case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'}) + case.assertTrue(obj.isdisjoint({'d', 'e'})) + case.assertFalse(obj.isdisjoint({'a', 'd'})) + + case.assertTrue(obj.issubset({'a', 'b', 'c', 'd'})) + case.assertFalse(obj.issubset({'a', 'b'})) + case.assertLess(obj, {'a', 'b', 'c', 'd'}) + case.assertLessEqual(obj, {'a', 'b', 'c'}) + + case.assertTrue(obj.issuperset({'a', 'b'})) + case.assertFalse(obj.issuperset({'a', 'b', 'd'})) + case.assertGreater(obj, {'a'}) + case.assertGreaterEqual(obj, {'a', 'b'}) + + def test_set(self): + o = self.manager.set() + self.run_worker(self._test_set_operator_symbols, o) + o = self.manager.set() + self.run_worker(self._test_set_operator_methods, o) + o = self.manager.set() + self.run_worker(self._test_set_comparisons, o) + + def test_set_init(self): + o = self.manager.set({'a', 'b', 'c'}) + self.assertSetEqual(o, {'a', 'b', 'c'}) + o = self.manager.set(["a", "b", "c"]) + self.assertSetEqual(o, {"a", "b", "c"}) + o = self.manager.set({"a": 1, "b": 2, "c": 3}) + self.assertSetEqual(o, {"a", "b", "c"}) + self.assertRaises(RemoteError, self.manager.set, 1234) + + def test_set_contain_all_method(self): + o = self.manager.set() + set_methods = { + '__and__', '__class_getitem__', '__contains__', '__iand__', '__ior__', + '__isub__', '__iter__', '__ixor__', '__len__', '__or__', '__rand__', + '__ror__', '__rsub__', '__rxor__', '__sub__', '__xor__', + '__ge__', '__gt__', '__le__', '__lt__', + 'add', 'clear', 'copy', 'difference', 'difference_update', 'discard', + 'intersection', 'intersection_update', 'isdisjoint', 'issubset', + 'issuperset', 'pop', 'remove', 'symmetric_difference', + 'symmetric_difference_update', 'union', 'update', + } + self.assertLessEqual(set_methods, set(dir(o))) + class TestNamedResource(unittest.TestCase): @only_run_in_spawn_testsuite("spawn specific test.") diff --git a/Misc/NEWS.d/next/Library/2025-02-11-06-42-17.gh-issue-129948.ZcugY9.rst b/Misc/NEWS.d/next/Library/2025-02-11-06-42-17.gh-issue-129948.ZcugY9.rst new file mode 100644 index 00000000000..85f7f966d83 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-02-11-06-42-17.gh-issue-129948.ZcugY9.rst @@ -0,0 +1,2 @@ +Add support for shared :class:`set` to :class:`multiprocessing.managers.SyncManager` +via :meth:`SyncManager.set() `.