diff --git a/src/pool/pool_disjoint.c b/src/pool/pool_disjoint.c index b241cc198c..52fbde1470 100644 --- a/src/pool/pool_disjoint.c +++ b/src/pool/pool_disjoint.c @@ -5,68 +5,7 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "critnib/critnib.h" -#include "uthash/utlist.h" - -#include "base_alloc_global.h" -#include "provider_tracking.h" -#include "utils_common.h" -#include "utils_concurrency.h" -#include "utils_log.h" -#include "utils_math.h" -#include "utils_sanitizers.h" - -typedef struct bucket_t bucket_t; -typedef struct slab_t slab_t; -typedef struct slab_list_item_t slab_list_item_t; -typedef struct disjoint_pool_t disjoint_pool_t; - -slab_t *create_slab(bucket_t *bucket); -void destroy_slab(slab_t *slab); - -void *slab_get(const slab_t *slab); -void *slab_get_end(const slab_t *slab); -void *slab_get_chunk(slab_t *slab); - -bool slab_has_avail(const slab_t *slab); -void slab_free_chunk(slab_t *slab, void *ptr); - -void slab_reg(slab_t *slab); -void slab_reg_by_addr(void *addr, slab_t *slab); -void slab_unreg(slab_t *slab); -void slab_unreg_by_addr(void *addr, slab_t *slab); - -bucket_t *create_bucket(size_t sz, disjoint_pool_t *pool, - umf_disjoint_pool_shared_limits_t *shared_limits); -void destroy_bucket(bucket_t *bucket); - -void bucket_update_stats(bucket_t *bucket, int in_use, int in_pool); -bool bucket_can_pool(bucket_t *bucket, bool *to_pool); -void bucket_decrement_pool(bucket_t *bucket, bool *from_pool); -void *bucket_get_chunk(bucket_t *bucket, bool *from_pool); -size_t bucket_chunk_cut_off(bucket_t *bucket); -size_t bucket_capacity(bucket_t *bucket); -void bucket_free_chunk(bucket_t *bucket, void *ptr, slab_t *slab, - bool *to_pool); -void bucket_count_alloc(bucket_t *bucket, bool from_pool); - -void *bucket_get_slab(bucket_t *bucket, bool *from_pool); -size_t bucket_slab_alloc_size(bucket_t *bucket); -size_t bucket_slab_min_size(bucket_t *bucket); -slab_list_item_t *bucket_get_avail_slab(bucket_t *bucket, bool *from_pool); -slab_list_item_t *bucket_get_avail_full_slab(bucket_t *bucket, bool *from_pool); -void bucket_free_slab(bucket_t *bucket, slab_t *slab, bool *to_pool); +#include "pool_disjoint_internal.h" static __TLS umf_result_t TLS_last_allocation_error; @@ -113,119 +52,6 @@ static size_t CutOff = (size_t)1 << 31; // 2GB #endif } -typedef struct bucket_t { - size_t size; - - // Linked list of slabs which have at least 1 available chunk. - slab_list_item_t *available_slabs; - - // Linked list of slabs with 0 available chunk. - slab_list_item_t *unavailable_slabs; - - // Protects the bucket and all the corresponding slabs - utils_mutex_t bucket_lock; - - // Reference to the allocator context, used access memory allocation - // routines, slab map and etc. - disjoint_pool_t *pool; - - umf_disjoint_pool_shared_limits_t *shared_limits; - - // For buckets used in chunked mode, a counter of slabs in the pool. - // For allocations that use an entire slab each, the entries in the Available - // list are entries in the pool.Each slab is available for a new - // allocation.The size of the Available list is the size of the pool. - // For allocations that use slabs in chunked mode, slabs will be in the - // Available list if any one or more of their chunks is free.The entire slab - // is not necessarily free, just some chunks in the slab are free. To - // implement pooling we will allow one slab in the Available list to be - // entirely empty. Normally such a slab would have been freed. But - // now we don't, and treat this slab as "in the pool". - // When a slab becomes entirely free we have to decide whether to return it - // to the provider or keep it allocated. A simple check for size of the - // Available list is not sufficient to check whether any slab has been - // pooled yet.We would have to traverse the entire Available listand check - // if any of them is entirely free. Instead we keep a counter of entirely - // empty slabs within the Available list to speed up the process of checking - // if a slab in this bucket is already pooled. - size_t chunked_slabs_in_pool; - - // Statistics - size_t alloc_pool_count; - size_t free_count; - size_t curr_slabs_in_use; - size_t curr_slabs_in_pool; - size_t max_slabs_in_pool; - size_t alloc_count; - size_t max_slabs_in_use; -} bucket_t; - -// Represents the allocated memory block of size 'slab_min_size' -// Internally, it splits the memory block into chunks. The number of -// chunks depends of the size of a Bucket which created the Slab. -// Note: Bucket's methods are responsible for thread safety of Slab access, -// so no locking happens here. -typedef struct slab_t { - // Pointer to the allocated memory of slab_min_size bytes - void *mem_ptr; - size_t slab_size; - - // Represents the current state of each chunk: if the bit is set then the - // chunk is allocated, and if the chunk is free for allocation otherwise - bool *chunks; - size_t num_chunks; - - // Total number of allocated chunks at the moment. - size_t num_allocated; - - // The bucket which the slab belongs to - bucket_t *bucket; - - // Hints where to start search for free chunk in a slab - size_t first_free_chunk_idx; - - // Store iterator to the corresponding node in avail/unavail list - // to achieve O(1) removal - slab_list_item_t *iter; -} slab_t; - -typedef struct slab_list_item_t { - slab_t *val; - struct slab_list_item_t *prev, *next; -} slab_list_item_t; - -typedef struct umf_disjoint_pool_shared_limits_t { - size_t max_size; - size_t total_size; // requires atomic access -} umf_disjoint_pool_shared_limits_t; - -typedef struct disjoint_pool_t { - // It's important for the map to be destroyed last after buckets and their - // slabs This is because slab's destructor removes the object from the map. - critnib *known_slabs; // (void *, slab_t *) - - // TODO: prev std::shared_timed_mutex - ok? - utils_mutex_t known_slabs_map_lock; - - // Handle to the memory provider - umf_memory_provider_handle_t provider; - - // Array of bucket_t* - bucket_t **buckets; - size_t buckets_num; - - // Configuration for this instance - umf_disjoint_pool_params_t params; - - umf_disjoint_pool_shared_limits_t *default_shared_limits; - - // Used in algorithm for finding buckets - size_t min_bucket_size_exp; - - // Coarse-grain allocation min alignment - size_t provider_min_page_size; -} disjoint_pool_t; - slab_t *create_slab(bucket_t *bucket) { assert(bucket); @@ -331,7 +157,7 @@ void *slab_get_chunk(slab_t *slab) { slab->num_allocated += 1; // use the found index as the next hint - slab->first_free_chunk_idx = chunk_idx; + slab->first_free_chunk_idx = chunk_idx + 1; return free_chunk; } diff --git a/src/pool/pool_disjoint_internal.h b/src/pool/pool_disjoint_internal.h new file mode 100644 index 0000000000..a46bb18ba8 --- /dev/null +++ b/src/pool/pool_disjoint_internal.h @@ -0,0 +1,189 @@ +/* + * Copyright (C) 2022-2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#ifndef UMF_POOL_DISJOINT_INTERNAL_H +#define UMF_POOL_DISJOINT_INTERNAL_H 1 + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "critnib/critnib.h" +#include "uthash/utlist.h" + +#include "base_alloc_global.h" +#include "provider_tracking.h" +#include "utils_common.h" +#include "utils_concurrency.h" +#include "utils_log.h" +#include "utils_math.h" +#include "utils_sanitizers.h" + +typedef struct bucket_t bucket_t; +typedef struct slab_t slab_t; +typedef struct slab_list_item_t slab_list_item_t; +typedef struct disjoint_pool_t disjoint_pool_t; + +typedef struct bucket_t { + size_t size; + + // Linked list of slabs which have at least 1 available chunk. + slab_list_item_t *available_slabs; + + // Linked list of slabs with 0 available chunk. + slab_list_item_t *unavailable_slabs; + + // Protects the bucket and all the corresponding slabs + utils_mutex_t bucket_lock; + + // Reference to the allocator context, used access memory allocation + // routines, slab map and etc. + disjoint_pool_t *pool; + + umf_disjoint_pool_shared_limits_t *shared_limits; + + // For buckets used in chunked mode, a counter of slabs in the pool. + // For allocations that use an entire slab each, the entries in the Available + // list are entries in the pool.Each slab is available for a new + // allocation.The size of the Available list is the size of the pool. + // For allocations that use slabs in chunked mode, slabs will be in the + // Available list if any one or more of their chunks is free.The entire slab + // is not necessarily free, just some chunks in the slab are free. To + // implement pooling we will allow one slab in the Available list to be + // entirely empty. Normally such a slab would have been freed. But + // now we don't, and treat this slab as "in the pool". + // When a slab becomes entirely free we have to decide whether to return it + // to the provider or keep it allocated. A simple check for size of the + // Available list is not sufficient to check whether any slab has been + // pooled yet.We would have to traverse the entire Available listand check + // if any of them is entirely free. Instead we keep a counter of entirely + // empty slabs within the Available list to speed up the process of checking + // if a slab in this bucket is already pooled. + size_t chunked_slabs_in_pool; + + // Statistics + size_t alloc_pool_count; + size_t free_count; + size_t curr_slabs_in_use; + size_t curr_slabs_in_pool; + size_t max_slabs_in_pool; + size_t alloc_count; + size_t max_slabs_in_use; +} bucket_t; + +// Represents the allocated memory block of size 'slab_min_size' +// Internally, it splits the memory block into chunks. The number of +// chunks depends of the size of a Bucket which created the Slab. +// Note: Bucket's methods are responsible for thread safety of Slab access, +// so no locking happens here. +typedef struct slab_t { + // Pointer to the allocated memory of slab_min_size bytes + void *mem_ptr; + size_t slab_size; + + // Represents the current state of each chunk: if the bit is set then the + // chunk is allocated, and if the chunk is free for allocation otherwise + bool *chunks; + size_t num_chunks; + + // Total number of allocated chunks at the moment. + size_t num_allocated; + + // The bucket which the slab belongs to + bucket_t *bucket; + + // Hints where to start search for free chunk in a slab + size_t first_free_chunk_idx; + + // Store iterator to the corresponding node in avail/unavail list + // to achieve O(1) removal + slab_list_item_t *iter; +} slab_t; + +typedef struct slab_list_item_t { + slab_t *val; + struct slab_list_item_t *prev, *next; +} slab_list_item_t; + +typedef struct umf_disjoint_pool_shared_limits_t { + size_t max_size; + size_t total_size; // requires atomic access +} umf_disjoint_pool_shared_limits_t; + +typedef struct disjoint_pool_t { + // It's important for the map to be destroyed last after buckets and their + // slabs This is because slab's destructor removes the object from the map. + critnib *known_slabs; // (void *, slab_t *) + + // TODO: prev std::shared_timed_mutex - ok? + utils_mutex_t known_slabs_map_lock; + + // Handle to the memory provider + umf_memory_provider_handle_t provider; + + // Array of bucket_t* + bucket_t **buckets; + size_t buckets_num; + + // Configuration for this instance + umf_disjoint_pool_params_t params; + + umf_disjoint_pool_shared_limits_t *default_shared_limits; + + // Used in algorithm for finding buckets + size_t min_bucket_size_exp; + + // Coarse-grain allocation min alignment + size_t provider_min_page_size; +} disjoint_pool_t; + +slab_t *create_slab(bucket_t *bucket); +void destroy_slab(slab_t *slab); + +void *slab_get(const slab_t *slab); +void *slab_get_end(const slab_t *slab); +void *slab_get_chunk(slab_t *slab); + +bool slab_has_avail(const slab_t *slab); +void slab_free_chunk(slab_t *slab, void *ptr); + +void slab_reg(slab_t *slab); +void slab_reg_by_addr(void *addr, slab_t *slab); +void slab_unreg(slab_t *slab); +void slab_unreg_by_addr(void *addr, slab_t *slab); + +bucket_t *create_bucket(size_t sz, disjoint_pool_t *pool, + umf_disjoint_pool_shared_limits_t *shared_limits); +void destroy_bucket(bucket_t *bucket); + +void bucket_update_stats(bucket_t *bucket, int in_use, int in_pool); +bool bucket_can_pool(bucket_t *bucket, bool *to_pool); +void bucket_decrement_pool(bucket_t *bucket, bool *from_pool); +void *bucket_get_chunk(bucket_t *bucket, bool *from_pool); +size_t bucket_chunk_cut_off(bucket_t *bucket); +size_t bucket_capacity(bucket_t *bucket); +void bucket_free_chunk(bucket_t *bucket, void *ptr, slab_t *slab, + bool *to_pool); +void bucket_count_alloc(bucket_t *bucket, bool from_pool); + +void *bucket_get_slab(bucket_t *bucket, bool *from_pool); +size_t bucket_slab_alloc_size(bucket_t *bucket); +size_t bucket_slab_min_size(bucket_t *bucket); +slab_list_item_t *bucket_get_avail_slab(bucket_t *bucket, bool *from_pool); +slab_list_item_t *bucket_get_avail_full_slab(bucket_t *bucket, bool *from_pool); +void bucket_free_slab(bucket_t *bucket, slab_t *slab, bool *to_pool); + +bucket_t *disjoint_pool_find_bucket(disjoint_pool_t *pool, size_t size); + +#endif // UMF_POOL_DISJOINT_INTERNAL_H diff --git a/test/pools/disjoint_pool.cpp b/test/pools/disjoint_pool.cpp index 5f048a7e60..9471f5809f 100644 --- a/test/pools/disjoint_pool.cpp +++ b/test/pools/disjoint_pool.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "pool.hpp" +#include "pool/pool_disjoint_internal.h" #include "poolFixtures.hpp" #include "pool_disjoint.h" #include "provider.hpp" @@ -21,6 +22,98 @@ umf_disjoint_pool_params_t poolConfig() { using umf_test::test; using namespace umf_test; +TEST_F(test, internals) { + static umf_result_t expectedResult = UMF_RESULT_SUCCESS; + struct memory_provider : public umf_test::provider_base_t { + umf_result_t alloc(size_t size, size_t, void **ptr) noexcept { + *ptr = malloc(size); + return UMF_RESULT_SUCCESS; + } + + umf_result_t free(void *ptr, [[maybe_unused]] size_t size) noexcept { + // do the actual free only when we expect the success + if (expectedResult == UMF_RESULT_SUCCESS) { + ::free(ptr); + } + return expectedResult; + } + + umf_result_t + get_min_page_size([[maybe_unused]] void *ptr, + [[maybe_unused]] size_t *pageSize) noexcept { + *pageSize = 1024; + return UMF_RESULT_SUCCESS; + } + }; + umf_memory_provider_ops_t provider_ops = + umf::providerMakeCOps(); + + auto providerUnique = + wrapProviderUnique(createProviderChecked(&provider_ops, nullptr)); + + umf_memory_provider_handle_t provider_handle; + provider_handle = providerUnique.get(); + + umf_disjoint_pool_params_t params = poolConfig(); + // set to maximum tracing + params.PoolTrace = 3; + + // in "internals" test we use ops interface to directly manipulate the pool + // structure + umf_memory_pool_ops_t *ops = umfDisjointPoolOps(); + EXPECT_NE(ops, nullptr); + + disjoint_pool_t *pool; + umf_result_t res = + ops->initialize(provider_handle, ¶ms, (void **)&pool); + EXPECT_EQ(res, UMF_RESULT_SUCCESS); + EXPECT_NE(pool, nullptr); + EXPECT_EQ(pool->provider_min_page_size, 1024); + + // test small allocations + size_t size = 8; + void *ptr = ops->malloc(pool, size); + EXPECT_NE(ptr, nullptr); + + // get bucket - because of small size this should be the first bucket in + // the pool + bucket_t *bucket = pool->buckets[0]; + EXPECT_NE(bucket, nullptr); + + // check bucket stats + EXPECT_EQ(bucket->alloc_count, 1); + EXPECT_EQ(bucket->alloc_pool_count, 1); + EXPECT_EQ(bucket->curr_slabs_in_use, 1); + + // check slab - there should be only single slab allocated + EXPECT_NE(bucket->available_slabs, nullptr); + EXPECT_EQ(bucket->available_slabs->next, nullptr); + slab_t *slab = bucket->available_slabs->val; + + // check slab stats + EXPECT_GE(slab->slab_size, params.SlabMinSize); + EXPECT_GE(slab->num_chunks, slab->slab_size / bucket->size); + + // check allocation in slab + EXPECT_EQ(slab->chunks[0], true); + EXPECT_EQ(slab->chunks[1], false); + EXPECT_EQ(slab->first_free_chunk_idx, 1); + + // TODO: + // * multiple alloc + free from single bucket + // * alignments + // * full slab alloc + // * slab overflow + // * chunked slabs + // * multiple alloc + free from different buckets + // * alloc something outside pool (> MaxPoolableSize) + // * test capacity + // * check minBucketSize + + // cleanup + ops->finalize(pool); +} + TEST_F(test, freeErrorPropagation) { static umf_result_t expectedResult = UMF_RESULT_SUCCESS; struct memory_provider : public umf_test::provider_base_t {