Merge master

src/benchmark.h

@@ -27,6 +27,16 @@ namespace Stockfish::Benchmark {
 
 std::vector<std::string> setup_bench(const std::string&, std::istream&);
 
+struct BenchmarkSetup {
+    int                      ttSize;
+    int                      threads;
+    std::vector<std::string> commands;
+    std::string              originalInvocation;
+    std::string              filledInvocation;
+};
+
+BenchmarkSetup setup_benchmark(std::istream&);
+
 }  // namespace Stockfish
 
 #endif  // #ifndef BENCHMARK_H_INCLUDED

src/bitboard.cpp

@@ -24,7 +24,9 @@
 
 #include <set>
 #include <utility>
-#include "magics.h"
+#ifndef USE_PEXT
+    #include "magics.h"
+#endif
 
 namespace Stockfish {
 
@@ -60,7 +62,7 @@ const std::set<Direction> BishopDirections{2 * NORTH_EAST, 2 * SOUTH_EAST, 2 * S
 
 
 template<PieceType pt>
-void init_magics(Bitboard table[], Magic magics[], const Bitboard magicsInit[]);
+void init_magics(Bitboard table[], Magic magics[] IF_NOT_PEXT(, const Bitboard magicsInit[]));
 
 template<PieceType pt>
 Bitboard lame_leaper_path(Direction d, Square s);
@@ -107,11 +109,11 @@ void Bitboards::init() {
         for (Square s2 = SQ_A0; s2 <= SQ_I9; ++s2)
             SquareDistance[s1][s2] = std::max(distance<File>(s1, s2), distance<Rank>(s1, s2));
 
-    init_magics<ROOK>(RookTable, RookMagics, RookMagicsInit);
-    init_magics<CANNON>(CannonTable, CannonMagics, RookMagicsInit);
-    init_magics<BISHOP>(BishopTable, BishopMagics, BishopMagicsInit);
-    init_magics<KNIGHT>(KnightTable, KnightMagics, KnightMagicsInit);
-    init_magics<KNIGHT_TO>(KnightToTable, KnightToMagics, KnightToMagicsInit);
+    init_magics<ROOK>(RookTable, RookMagics IF_NOT_PEXT(, RookMagicsInit));
+    init_magics<CANNON>(CannonTable, CannonMagics IF_NOT_PEXT(, RookMagicsInit));
+    init_magics<BISHOP>(BishopTable, BishopMagics IF_NOT_PEXT(, BishopMagicsInit));
+    init_magics<KNIGHT>(KnightTable, KnightMagics IF_NOT_PEXT(, KnightMagicsInit));
+    init_magics<KNIGHT_TO>(KnightToTable, KnightToMagics IF_NOT_PEXT(, KnightToMagicsInit));
 
     for (Square s1 = SQ_A0; s1 <= SQ_I9; ++s1)
     {
@@ -238,10 +240,10 @@ Bitboard lame_leaper_attack(Square s, Bitboard occupied) {
 
 // Computes all rook and bishop attacks at startup. Magic
 // bitboards are used to look up attacks of sliding pieces. As a reference see
-// www.chessprogramming.org/Magic_Bitboards. In particular, here we use the so
-// called "fancy" approach.
+// https://www.chessprogramming.org/Magic_Bitboards. In particular, here we use
+// the so called "fancy" approach.
 template<PieceType pt>
-void init_magics(Bitboard table[], Magic magics[], const Bitboard magicsInit[]) {
+void init_magics(Bitboard table[], Magic magics[] IF_NOT_PEXT(, const Bitboard magicsInit[])) {
 
     Bitboard edges, b;
     uint64_t size = 0;
@@ -262,12 +264,12 @@ void init_magics(Bitboard table[], Magic magics[], const Bitboard magicsInit[])
         if (pt != KNIGHT_TO)
             m.mask &= ~edges;
 
-        if (HasPext)
-            m.shift = popcount(uint64_t(m.mask));
-        else
-            m.shift = 128 - popcount(m.mask);
-
+#ifdef USE_PEXT
+        m.shift = popcount(uint64_t(m.mask));
+#else
         m.magic = magicsInit[s];
+        m.shift = 128 - popcount(m.mask);
+#endif
 
         // Set the offset for the attacks table of the square. We have individual
         // table sizes for each square with "Fancy Magic Bitboards".

src/bitboard.h

@@ -28,6 +28,12 @@
 
 #include "types.h"
 
+#ifdef USE_PEXT
+    #define IF_NOT_PEXT(...)
+#else
+    #define IF_NOT_PEXT(...) __VA_ARGS__
+#endif
+
 namespace Stockfish {
 
 namespace Bitboards {
@@ -82,17 +88,18 @@ int popcount(Bitboard b);  // required for 128 bit pext
 // Magic holds all magic bitboards relevant data for a single square
 struct Magic {
     Bitboard  mask;
-    Bitboard  magic;
     Bitboard* attacks;
     unsigned  shift;
+    IF_NOT_PEXT(Bitboard magic;)
 
     // Compute the attack's index using the 'magic bitboards' approach
     unsigned index(Bitboard occupied) const {
 
-        if (HasPext)
-            return unsigned(pext(occupied, mask, shift));
-
+#ifdef USE_PEXT
+        return unsigned(pext(occupied, mask, shift));
+#else
         return unsigned(((occupied & mask) * magic) >> shift);
+#endif
     }
 };
 

src/engine.cpp

@@ -20,9 +20,7 @@
 
 #include <cassert>
 #include <deque>
-#include <iosfwd>
 #include <memory>
-#include <ostream>
 #include <sstream>
 #include <string_view>
 #include <utility>
@@ -45,8 +43,8 @@ namespace NN = Eval::NNUE;
 constexpr auto StartFEN  = "rnbakabnr/9/1c5c1/p1p1p1p1p/9/9/P1P1P1P1P/1C5C1/9/RNBAKABNR w";
 constexpr int  MaxHashMB = Is64Bit ? 33554432 : 2048;
 
-Engine::Engine(std::string path) :
-    binaryDirectory(CommandLine::get_binary_directory(path)),
+Engine::Engine(std::optional<std::string> path) :
+    binaryDirectory(path ? CommandLine::get_binary_directory(*path) : ""),
     numaContext(NumaConfig::from_system()),
     states(new std::deque<StateInfo>(1)),
     threads(),
@@ -61,12 +59,13 @@ Engine::Engine(std::string path) :
 
     options["NumaPolicy"] << Option("auto", [this](const Option& o) {
         set_numa_config_from_option(o);
-        return numa_config_information_as_string() + "\n" + thread_binding_information_as_string();
+        return numa_config_information_as_string() + "\n"
+             + thread_allocation_information_as_string();
     });
 
     options["Threads"] << Option(1, 1, 1024, [this](const Option&) {
         resize_threads();
-        return thread_binding_information_as_string();
+        return thread_allocation_information_as_string();
     });
 
     options["Hash"] << Option(16, 1, MaxHashMB, [this](const Option& o) {
@@ -130,6 +129,10 @@ void Engine::set_on_bestmove(std::function<void(std::string_view, std::string_vi
     updateContext.onBestmove = std::move(f);
 }
 
+void Engine::set_on_verify_networks(std::function<void(std::string_view)>&& f) {
+    onVerifyNetworks = std::move(f);
+}
+
 void Engine::wait_for_search_finished() { threads.main_thread()->wait_for_search_finished(); }
 
 void Engine::set_position(const std::string& fen, const std::vector<std::string>& moves) {
@@ -199,7 +202,7 @@ void Engine::set_ponderhit(bool b) { threads.main_manager()->ponder = b; }
 
 // network related
 
-void Engine::verify_network() const { network->verify(options["EvalFile"]); }
+void Engine::verify_network() const { network->verify(options["EvalFile"], onVerifyNetworks); }
 
 void Engine::load_network(const std::string& file) {
     network.modify_and_replicate(
@@ -237,6 +240,8 @@ std::string Engine::visualize() const {
     return ss.str();
 }
 
+int Engine::get_hashfull(int maxAge) const { return tt.hashfull(maxAge); }
+
 std::vector<std::pair<size_t, size_t>> Engine::get_bound_thread_count_by_numa_node() const {
     auto                                   counts = threads.get_bound_thread_count_by_numa_node();
     const NumaConfig&                      cfg    = numaContext.get_numa_config();
@@ -262,15 +267,9 @@ std::string Engine::numa_config_information_as_string() const {
 std::string Engine::thread_binding_information_as_string() const {
     auto              boundThreadsByNode = get_bound_thread_count_by_numa_node();
     std::stringstream ss;
-
-    size_t threadsSize = threads.size();
-    ss << "Using " << threadsSize << (threadsSize > 1 ? " threads" : " thread");
-
     if (boundThreadsByNode.empty())
         return ss.str();
 
-    ss << " with NUMA node thread binding: ";
-
     bool isFirst = true;
 
     for (auto&& [current, total] : boundThreadsByNode)
@@ -284,4 +283,20 @@ std::string Engine::thread_binding_information_as_string() const {
     return ss.str();
 }
 
+std::string Engine::thread_allocation_information_as_string() const {
+    std::stringstream ss;
+
+    size_t threadsSize = threads.size();
+    ss << "Using " << threadsSize << (threadsSize > 1 ? " threads" : " thread");
+
+    auto boundThreadsByNodeStr = thread_binding_information_as_string();
+    if (boundThreadsByNodeStr.empty())
+        return ss.str();
+
+    ss << " with NUMA node thread binding: ";
+    ss << boundThreadsByNodeStr;
+
+    return ss.str();
+}
+
 }

src/engine.h

@@ -25,6 +25,7 @@
 #include <optional>
 #include <string>
 #include <string_view>
+#include <utility>
 #include <vector>
 
 #include "nnue/network.h"
@@ -44,7 +45,7 @@ class Engine {
     using InfoFull  = Search::InfoFull;
     using InfoIter  = Search::InfoIteration;
 
-    Engine(std::string path = "");
+    Engine(std::optional<std::string> path = std::nullopt);
 
     // Cannot be movable due to components holding backreferences to fields
     Engine(const Engine&)            = delete;
@@ -78,6 +79,7 @@ class Engine {
     void set_on_update_full(std::function<void(const InfoFull&)>&&);
     void set_on_iter(std::function<void(const InfoIter&)>&&);
     void set_on_bestmove(std::function<void(std::string_view, std::string_view)>&&);
+    void set_on_verify_networks(std::function<void(std::string_view)>&&);
 
     // network related
 
@@ -92,12 +94,15 @@ class Engine {
     const OptionsMap& get_options() const;
     OptionsMap&       get_options();
 
+    int get_hashfull(int maxAge = 0) const;
+
     std::string                            fen() const;
     void                                   flip();
     std::string                            visualize() const;
     std::vector<std::pair<size_t, size_t>> get_bound_thread_count_by_numa_node() const;
     std::string                            get_numa_config_as_string() const;
     std::string                            numa_config_information_as_string() const;
+    std::string                            thread_allocation_information_as_string() const;
     std::string                            thread_binding_information_as_string() const;
 
    private:
@@ -114,7 +119,8 @@ class Engine {
     TranspositionTable                      tt;
     LazyNumaReplicated<Eval::NNUE::Network> network;
 
-    Search::SearchManager::UpdateContext updateContext;
+    Search::SearchManager::UpdateContext  updateContext;
+    std::function<void(std::string_view)> onVerifyNetworks;
 };
 
 }  // namespace Stockfish

src/evaluate.cpp

@@ -25,7 +25,6 @@
 #include <iomanip>
 #include <memory>
 #include <sstream>
-#include <tuple>
 
 #include "nnue/network.h"
 #include "nnue/nnue_misc.h"
@@ -46,21 +45,18 @@ Value Eval::evaluate(const Eval::NNUE::Network& network,
     assert(!pos.checkers());
 
     auto [psqt, positional] = network.evaluate(pos, &caches.cache);
-    Value nnue              = (1563 * psqt + 1633 * positional) / 1183;
+    Value nnue              = psqt + positional;
     int   nnueComplexity    = std::abs(psqt - positional);
 
     // Blend optimism and eval with nnue complexity
-    optimism += optimism * nnueComplexity / 550;
-    nnue -= nnue * nnueComplexity / 10129;
+    optimism += optimism * nnueComplexity / 464;
+    nnue -= nnue * nnueComplexity / 10662;
 
-    int mm = pos.major_material() / 39;
-    int v  = (nnue * (430 + mm) + optimism * (101 + mm)) / 575;
-
-    // Evaluation grain (to get more alpha-beta cuts) with randomization (for robustness)
-    v = (v / 16) * 16 - 1 + (pos.key() & 0x2);
+    int mm = pos.major_material() / 43;
+    int v  = (nnue * (426 + mm) + optimism * (81 + mm)) / 503;
 
     // Damp down the evaluation linearly when shuffling
-    v -= (v * pos.rule60_count()) / 244;
+    v -= (v * pos.rule60_count()) / 267;
 
     // Guarantee evaluation does not hit the mate range
     v = std::clamp(v, VALUE_MATED_IN_MAX_PLY + 1, VALUE_MATE_IN_MAX_PLY - 1);

src/external/common/allocations.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* This file provides custom allocation primitives
+ */
+
+#define ZSTD_DEPS_NEED_MALLOC
+#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
+
+#include "compiler.h" /* MEM_STATIC */
+#define ZSTD_STATIC_LINKING_ONLY
+#include "../zstd.h" /* ZSTD_customMem */
+
+#ifndef ZSTD_ALLOCATIONS_H
+    #define ZSTD_ALLOCATIONS_H
+
+/* custom memory allocation functions */
+
+MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) {
+    if (customMem.customAlloc)
+        return customMem.customAlloc(customMem.opaque, size);
+    return ZSTD_malloc(size);
+}
+
+MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) {
+    if (customMem.customAlloc)
+    {
+        /* calloc implemented as malloc+memset;
+         * not as efficient as calloc, but next best guess for custom malloc */
+        void* const ptr = customMem.customAlloc(customMem.opaque, size);
+        ZSTD_memset(ptr, 0, size);
+        return ptr;
+    }
+    return ZSTD_calloc(1, size);
+}
+
+MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) {
+    if (ptr != NULL)
+    {
+        if (customMem.customFree)
+            customMem.customFree(customMem.opaque, ptr);
+        else
+            ZSTD_free(ptr);
+    }
+}
+
+#endif /* ZSTD_ALLOCATIONS_H */