aligned_memory.hpp

Go to the documentation of this file.

//============================================================================
// vSMC/include/vsmc/utility/aligned_memory.hpp
//----------------------------------------------------------------------------
//                         vSMC: Scalable Monte Carlo
//----------------------------------------------------------------------------
// Copyright (c) 2013-2016, Yan Zhou
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//   Redistributions of source code must retain the above copyright notice,
//   this list of conditions and the following disclaimer.
//
//   Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//============================================================================

#ifndef VSMC_UTILITY_ALIGNED_MEMORY_HPP
#define VSMC_UTILITY_ALIGNED_MEMORY_HPP

#include <vsmc/internal/basic.hpp>

#if VSMC_HAS_POSIX
#include <stdlib.h>
#elif defined(VSMC_MSVC)
#include <malloc.h>
#endif

#if VSMC_HAS_TBB
#include <tbb/scalable_allocator.h>
#endif

#ifndef VSMC_ALIGNMENT
#define VSMC_ALIGNMENT 32
#endif

#ifndef VSMC_ALIGNMENT_MIN
#define VSMC_ALIGNMENT_MIN 16
#endif

#if VSMC_ALIGNMENT < VSMC_ALIGNMENT_MIN
#undef VSMC_ALIGNEMNT
#define VSMC_ALIGNMENT VSMC_ALIGNMENT_MIN
#endif

#ifndef VSMC_ALIGNED_MEMORY_TYPE
#if VSMC_USE_TBB_MALLOC
#define VSMC_ALIGNED_MEMORY_TYPE ::vsmc::AlignedMemoryTBB
#elif VSMC_HAS_POSIX || defined(VSMC_MSVC)
#define VSMC_ALIGNED_MEMORY_TYPE ::vsmc::AlignedMemorySYS
#else
#define VSMC_ALIGNED_MEMORY_TYPE ::vsmc::AlignedMemorySTD
#endif
#endif

#ifndef VSMC_CONSTRUCT_SCALAR
#define VSMC_CONSTRUCT_SCALAR 0
#endif

#define VSMC_DEFINE_NEW_DELETE(Class)                                         \
    public:                                                                   \
    static void *operator new(std::size_t n)                                  \
    {                                                                         \
        void *ptr = ::vsmc::AlignedMemory::aligned_malloc(                    \
            n, ::vsmc::AlignmentTrait<Class>::value);                         \
        if (ptr == nullptr)                                                   \
            throw std::bad_alloc();                                           \
                                                                              \
        return ptr;                                                           \
    }                                                                         \
                                                                              \
    static void *operator new[](std::size_t n)                                \
    {                                                                         \
        void *ptr = ::vsmc::AlignedMemory::aligned_malloc(                    \
            n, ::vsmc::AlignmentTrait<Class>::value);                         \
        if (ptr == nullptr)                                                   \
            throw std::bad_alloc();                                           \
                                                                              \
        return ptr;                                                           \
    }                                                                         \
                                                                              \
    static void *operator new(std::size_t, void *ptr) { return ptr; }         \
                                                                              \
    static void *operator new[](std::size_t, void *ptr) { return ptr; }       \
                                                                              \
    static void operator delete(void *ptr)                                    \
    {                                                                         \
        ::vsmc::AlignedMemory::aligned_free(ptr);                             \
    }                                                                         \
                                                                              \
    static void operator delete[](void *ptr)                                  \
    {                                                                         \
        ::vsmc::AlignedMemory::aligned_free(ptr);                             \
    }                                                                         \
                                                                              \
    static void operator delete(void *, void *) {}                            \
                                                                              \
    static void operator delete[](void *, void *) {}

namespace vsmc
{

namespace internal
{

template <typename T, bool = std::is_scalar<T>::value>
class AlignmentTraitImpl
{
    public:
    static constexpr std::size_t value =
        alignof(T) > VSMC_ALIGNMENT_MIN ? alignof(T) : VSMC_ALIGNMENT_MIN;
}; // class AlignmentTraitImpl

template <typename T>
class AlignmentTraitImpl<T, true>
{
    public:
    static constexpr std::size_t
        value = alignof(T) > VSMC_ALIGNMENT ? alignof(T) : VSMC_ALIGNMENT;
}; // class AlignmentTraitImpl

} // namespace vsmc::internal

template <typename T>
class AlignmentTrait : public std::integral_constant<std::size_t,
                           internal::AlignmentTraitImpl<T>::value>
{
}; // class AlignmentTrait

class AlignedMemorySTD
{
    public:
    static void *aligned_malloc(std::size_t n, std::size_t alignment) noexcept
    {
        std::size_t bytes = (n > 0 ? n : 1) + alignment + sizeof(void *);
        if (bytes < n)
            return nullptr;

        void *orig_ptr = std::malloc(bytes);
        if (orig_ptr == nullptr)
            return nullptr;

        uintptr_t address = reinterpret_cast<uintptr_t>(orig_ptr);
        uintptr_t offset = alignment - (address + sizeof(void *)) % alignment;
        void *ptr =
            reinterpret_cast<void *>(address + offset + sizeof(void *));
        void **orig = reinterpret_cast<void **>(address + offset);
        *orig = orig_ptr;

        return ptr;
    }

    static void aligned_free(void *ptr) noexcept
    {
        if (ptr != nullptr) {
            std::free(*reinterpret_cast<void **>(
                reinterpret_cast<uintptr_t>(ptr) - sizeof(void *)));
        }
    }
}; // class AlignedMemmorySTD

#if VSMC_HAS_POSIX

class AlignedMemorySYS
{
    public:
    static void *aligned_malloc(std::size_t n, std::size_t alignment) noexcept
    {
        void *ptr;
        if (posix_memalign(&ptr, alignment, n > 0 ? n : 1) != 0)
            return nullptr;
        return ptr;
    }

    static void aligned_free(void *ptr) noexcept
    {
        if (ptr != nullptr)
            free(ptr);
    }
}; // class AlignedMallocSYS

#elif defined(VSMC_MSVC)

class AlignedMemorySYS
{
    public:
    static void *aligned_malloc(std::size_t n, std::size_t alignment) noexcept
    {
        return _aligned_malloc(n > 0 ? n : 1, alignment);
    }

    static void aligned_free(void *ptr) noexcept
    {
        if (ptr != nullptr)
            _aligned_free(ptr);
    }
}; // class AlignedMemorySYS

#endif // VSMC_HAS_POSIX

#if VSMC_HAS_TBB

class AlignedMemoryTBB
{
    public:
    static void *aligned_malloc(std::size_t n, std::size_t alignment) noexcept
    {
        return scalable_aligned_malloc(n > 0 ? n : 1, alignment);
    }

    static void aligned_free(void *ptr) noexcept
    {
        if (ptr != nullptr)
            scalable_aligned_free(ptr);
    }
}; // class AlignedMemoryTBB

#endif // VSMC_HAS_TBB

using AlignedMemory = VSMC_ALIGNED_MEMORY_TYPE;

template <typename T, std::size_t Alignment = AlignmentTrait<T>::value,
    typename Memory = AlignedMemory>
class Allocator : public std::allocator<T>
{
    static_assert(Alignment != 0 && (Alignment & (Alignment - 1)) == 0,
        "**Allocator** USED WITH Alignment OTHER THAN A POWER OF TWO "
        "POSITIVE INTEGER");

    static_assert(Alignment >= sizeof(void *),
        "**Allocator** USED WITH Alignment LESS THAN sizeof(void *)");

    public:
    using value_type = T;
    using size_type = std::size_t;
    using difference_type = std::ptrdiff_t;
    using pointer = T *;
    using const_pointer = const T *;
    using reference = typename std::add_lvalue_reference<T>::type;
    using const_reference = typename std::add_lvalue_reference<const T>::type;
    using is_always_equal = std::true_type;

    template <typename U>
    class rebind
    {
        public:
        using other = Allocator<U, Alignment, Memory>;
    }; // class rebind

    Allocator() = default;

    Allocator(const Allocator<T, Alignment, Memory> &) = default;

    Allocator(Allocator<T, Alignment, Memory> &&) = default;

    Allocator<T, Alignment, Memory> &operator=(
        const Allocator<T, Alignment, Memory> &) = default;

    Allocator<T, Alignment, Memory> &operator=(
        Allocator<T, Alignment, Memory> &&) = default;

    template <typename U>
    Allocator(const Allocator<U, Alignment, Memory> &other)
        : std::allocator<T>(static_cast<std::allocator<U>>(other))
    {
    }

    template <typename U>
    Allocator(Allocator<U, Alignment, Memory> &&other)
        : std::allocator<T>(std::move(static_cast<std::allocator<U>>(other)))
    {
    }

    pointer allocate(size_type n, const void * = nullptr)
    {
        n = std::max(n, static_cast<size_type>(1));
        size_type bytes = n * sizeof(value_type);
        if (bytes < n)
            throw std::bad_alloc();

        pointer ptr =
            static_cast<pointer>(Memory::aligned_malloc(bytes, Alignment));
        if (ptr == nullptr)
            throw std::bad_alloc();

        return ptr;
    }

    void deallocate(pointer ptr, size_type = 0) noexcept
    {
        if (ptr != nullptr)
            Memory::aligned_free(ptr);
    }

    template <typename U>
    void construct(U *ptr)
    {
        construct_dispatch(ptr,
            std::integral_constant<bool, (VSMC_CONSTRUCT_SCALAR != 0 ||
                                             !std::is_scalar<U>::value)>());
    }

    template <typename U, typename Arg, typename... Args>
    void constrct(U *ptr, Arg &&arg, Args &&... args)
    {
        std::allocator<T>::construct(
            ptr, std::forward<Arg>(arg), std::forward<Args>(args)...);
    }

    private:
    template <typename U>
    void construct_dispatch(U *ptr, std::true_type)
    {
        std::allocator<T>::construct(ptr);
    }

    template <typename U>
    void construct_dispatch(U *, std::false_type)
    {
    }
}; // class Allocator

template <std::size_t Alignment, typename Memory>
class Allocator<void, Alignment, Memory>
{
    using value_type = void;
    using pointer = void *;
    using const_pointer = const void *;

    template <class U>
    struct rebind {
        using other = Allocator<U, Alignment, Memory>;
    };
}; // class Allocator

template <std::size_t Alignment, typename Memory>
class Allocator<const void, Alignment, Memory>
{
    using value_type = const void;
    using pointer = const void *;
    using const_pointer = const void *;

    template <class U>
    struct rebind {
        using other = Allocator<U, Alignment, Memory>;
    };
}; // class Allocator

template <typename T1, typename T2, std::size_t Alignment, typename Memory>
inline bool operator==(const Allocator<T1, Alignment, Memory> &,
    const Allocator<T2, Alignment, Memory> &)
{
    return true;
}

template <typename T1, typename T2, std::size_t Alignment, typename Memory>
inline bool operator!=(const Allocator<T1, Alignment, Memory> &,
    const Allocator<T2, Alignment, Memory> &)
{
    return false;
}

namespace internal
{

template <typename Alloc>
class AllocatorAlignment
    : public AlignmentTrait<typename std::allocator_traits<Alloc>::value_type>
{
}; // class AllocatorAlignemnt

template <typename T, std::size_t Alignment, typename Memory>
class AllocatorAlignment<Allocator<T, Alignment, Memory>>
    : public std::integral_constant<std::size_t, Alignment>
{
}; // class AllocatorAlignment

} // namespace vsmc::internal

template <typename T, typename Alloc = Allocator<T>>
using Vector = std::vector<T, Alloc>;

template <typename T, std::size_t N,
    std::size_t Alignment = AlignmentTrait<T>::value>
class alignas(Alignment) Array : public std::array<T, N>
{
    static_assert(Alignment != 0 && (Alignment & (Alignment - 1)) == 0,
        "**Array** USED WITH Alignment OTHER THAN A POWER OF TWO "
        "POSITIVE INTEGER");

    static_assert(Alignment >= sizeof(void *),
        "**Array** USED WITH Alignment LESS THAN sizeof(void *)");
}; // class Array

} // namespace vsmc

#endif // VSMC_UTILITY_ALIGNED_MEMORY_HPP