P4C
The P4 Compiler
Loading...
Searching...
No Matches
tofino/input_xbar.h
1
19#ifndef BF_P4C_MAU_TOFINO_INPUT_XBAR_H_
20#define BF_P4C_MAU_TOFINO_INPUT_XBAR_H_
21
22/* clang-format off */
23#include "backends/tofino/bf-p4c/mau/input_xbar.h"
24#include "backends/tofino/bf-p4c/common/alloc.h"
25
26namespace Tofino {
27
28using namespace P4;
29
30struct IXBar : public ::IXBar {
31 // FIXME -- make these per-device params
32 static constexpr int EXACT_GROUPS = 8;
33 static constexpr int EXACT_BYTES_PER_GROUP = 16;
34 static constexpr int HASH_TABLES = 16;
35 static constexpr int HASH_GROUPS = 8;
36 static constexpr int HASH_INDEX_GROUPS = 4; /* groups of 10 bits for indexing */
37 static constexpr int HASH_SINGLE_BITS = 12; /* top 12 bits of hash table individually */
38 static constexpr int HASH_PARITY_BIT = 51; /* If enabled reserved parity bit position */
39 static constexpr int RAM_SELECT_BIT_START = 40;
40 static constexpr int RAM_LINE_SELECT_BITS = 10;
41 static constexpr int HASH_MATRIX_SIZE = RAM_SELECT_BIT_START + HASH_SINGLE_BITS;
42 static constexpr int HASH_DIST_SLICES = 3;
43 static constexpr int HASH_DIST_BITS = 16;
44 static constexpr int HASH_DIST_EXPAND_BITS = 7;
45 static constexpr int HASH_DIST_MAX_MASK_BITS = HASH_DIST_BITS + HASH_DIST_EXPAND_BITS;
46 static constexpr int HASH_DIST_UNITS = 2;
47 static constexpr int TOFINO_METER_ALU_BYTE_OFFSET = 8;
48 static constexpr int LPF_INPUT_BYTES = 4;
49 static constexpr int TERNARY_GROUPS = StageUse::MAX_TERNARY_GROUPS;
50 static constexpr int BYTE_GROUPS = StageUse::MAX_TERNARY_GROUPS/2;
51 static constexpr int TERNARY_BYTES_PER_GROUP = 5;
52 static constexpr int TERNARY_BYTES_PER_BIG_GROUP = 11;
53 static constexpr int GATEWAY_SEARCH_BYTES = 4;
54 static constexpr int RESILIENT_MODE_HASH_BITS = 51;
55 static constexpr int FAIR_MODE_HASH_BITS = 14;
56 static constexpr int METER_ALU_HASH_BITS = 52;
57 static constexpr int METER_ALU_HASH_PARITY_BYTE_START = 48;
58 static constexpr int METER_PRECOLOR_SIZE = 2;
59 static constexpr int REPEATING_CONSTRAINT_SECT = 4;
60 static constexpr int MAX_HASH_BITS = 52;
61 static constexpr le_bitrange SELECT_BIT_RANGE =
62 le_bitrange(RAM_SELECT_BIT_START, METER_ALU_HASH_BITS-1);
63 static constexpr le_bitrange INDEX_BIT_RANGE(int group) {
64 return le_bitrange(group*RAM_LINE_SELECT_BITS, (group+1)*RAM_LINE_SELECT_BITS - 1); }
65 static constexpr int INDEX_RANGE_SUBGROUP(le_bitrange r) { return r.lo / RAM_LINE_SELECT_BITS; }
66
67 static int get_meter_alu_hash_bits() {
68 // If hash parity is enabled reserve a bit for parity
69 if (!BackendOptions().disable_gfm_parity)
70 return METER_ALU_HASH_BITS - 1;
71 return METER_ALU_HASH_BITS;
72 }
73
74 static int get_hash_single_bits() {
75 static bool disable_gfm_parity = BackendOptions().disable_gfm_parity;
76 // If hash parity is enabled reserve a bit for parity
77 if (!disable_gfm_parity)
78 return HASH_SINGLE_BITS - 1;
79 return HASH_SINGLE_BITS;
80 }
81
82 private:
83 static int get_hash_matrix_size() {
84 return RAM_SELECT_BIT_START + get_hash_single_bits();
85 }
86
87 static int get_max_hash_bits() {
88 // If hash parity is enabled reserve a bit for parity
89 if (!BackendOptions().disable_gfm_parity)
90 return MAX_HASH_BITS - 1;
91 return MAX_HASH_BITS;
92 }
93
94 using Loc = ::IXBar::Loc;
95 using byte_speciality_t = ::IXBar::byte_speciality_t;
96 // FIXME -- should have a better way of importing all the byte_speciality_t values
97 static constexpr auto NONE = ::IXBar::NONE;
98 static constexpr auto ATCAM_DOUBLE = ::IXBar::ATCAM_DOUBLE;
99 static constexpr auto ATCAM_INDEX = ::IXBar::ATCAM_INDEX;
100 static constexpr auto RANGE_LO = ::IXBar::RANGE_LO;
101 static constexpr auto RANGE_HI = ::IXBar::RANGE_HI;
102 static constexpr auto BYTE_SPECIALITIES = ::IXBar::BYTE_SPECIALITIES;
103
104 using FieldInfo = ::IXBar::FieldInfo;
105 using parity_status_t = ::IXBar::parity_status_t;
106 // FIXME -- should have a better waby of importing all the parity_status_t values
107 static constexpr auto PARITY_NONE = ::IXBar::PARITY_NONE;
108 static constexpr auto PARITY_ENABLED = ::IXBar::PARITY_ENABLED;
109 static constexpr auto PARITY_DISABLED = ::IXBar::PARITY_DISABLED;
110
114 BFN::Alloc2D<std::pair<PHV::Container, int>, EXACT_GROUPS, EXACT_BYTES_PER_GROUP>
115 exact_use;
116 BFN::Alloc2D<std::pair<PHV::Container, int>, TERNARY_GROUPS, TERNARY_BYTES_PER_GROUP>
117 ternary_use;
118 BFN::Alloc1D<std::pair<PHV::Container, int>, BYTE_GROUPS> byte_group_use;
119 BFN::Alloc2Dbase<std::pair<PHV::Container, int>> &use(bool ternary) {
120 if (ternary) return ternary_use;
121 return exact_use; }
122 /* reverse maps of the above, mapping containers sets of group+byte */
123 std::multimap<PHV::Container, Loc> exact_fields;
124 std::multimap<PHV::Container, Loc> ternary_fields;
125 std::multimap<PHV::Container, Loc> &fields(bool ternary) {
126 return ternary ? ternary_fields : exact_fields; }
127
128 // map from container to tables that use those fields (mostly for debugging)
129 std::map<PHV::Container, std::set<cstring>> field_users;
130
131 /* Track the use of hashtables/groups too -- FIXME -- should it be a separate data structure?
132 * strings here are table names
133 * NOTE: Changes here require changes to .gdbinit pretty printer */
134 unsigned hash_index_inuse[HASH_INDEX_GROUPS] = { 0 };
135 BFN::Alloc2D<cstring, HASH_TABLES, HASH_INDEX_GROUPS> hash_index_use;
136 BFN::Alloc2D<cstring, HASH_TABLES, HASH_SINGLE_BITS> hash_single_bit_use;
137 unsigned hash_single_bit_inuse[HASH_SINGLE_BITS] = { 0 };
138 BFN::Alloc1D<cstring, HASH_GROUPS> hash_group_print_use;
139 unsigned hash_group_use[HASH_GROUPS] = { 0 };
140 BFN::Alloc2D<cstring, HASH_TABLES, HASH_DIST_SLICES> hash_dist_use;
141 bitvec hash_dist_inuse[HASH_TABLES] = { bitvec() };
142 BFN::Alloc2D<cstring, HASH_TABLES, HASH_DIST_SLICES * HASH_DIST_BITS> hash_dist_bit_use;
143 bitvec hash_dist_bit_inuse[HASH_TABLES] = { bitvec() };
144 parity_status_t hash_group_parity_use[HASH_GROUPS] = { PARITY_NONE };
145
146 // Added on update to be verified
147 bitvec hash_used_per_function[HASH_GROUPS] = { bitvec() };
148
149 int hash_dist_groups[HASH_DIST_UNITS] = {-1, -1};
150 friend class IXBarRealign;
151
152 /* API for unit tests */
153 public:
154 BFN::Alloc2Dbase<std::pair<PHV::Container, int>> &get_exact_use() { return exact_use; }
155 BFN::Alloc2Dbase<std::pair<PHV::Container, int>> &get_ternary_use() { return ternary_use; }
156 BFN::Alloc1D<std::pair<PHV::Container, int>, BYTE_GROUPS> &get_byte_group_use() {
157 return byte_group_use; }
158
159 // map (type, group, byte) coordinate to linear xbar output space
160 unsigned toIXBarOutputByte(bool ternary, int group, int byte) {
161 unsigned offset = 0;
162 if (ternary) {
163 unsigned mid_byte_count = (group / 2) + (group % 2);
164 offset = (group / 2) * 10 + (group % 2) * 5 + byte + mid_byte_count;
165 } else {
166 offset = group * 16 + byte;
167 }
168 return offset;
169 }
170
171 private:
172 std::set<cstring> dleft_updates;
173
174 public:
175 using byte_type_t = ::IXBar::byte_type_t;
176 // FIXME -- should have a better waby of importing all the byte_type_t values
177 static constexpr auto NO_BYTE_TYPE = ::IXBar::NO_BYTE_TYPE;
178 static constexpr auto ATCAM = ::IXBar::ATCAM;
179 static constexpr auto PARTITION_INDEX = ::IXBar::PARTITION_INDEX;
180 static constexpr auto RANGE = ::IXBar::RANGE;
181 using HashDistDest_t = ::IXBar::HashDistDest_t;
182 // FIXME -- should have a better waby of importing all the HashDistDest_t values
183 static constexpr auto HD_IMMED_LO = ::IXBar::HD_IMMED_LO;
184 static constexpr auto HD_IMMED_HI = ::IXBar::HD_IMMED_HI;
185 static constexpr auto HD_STATS_ADR = ::IXBar::HD_STATS_ADR;
186 static constexpr auto HD_METER_ADR = ::IXBar::HD_METER_ADR;
187 static constexpr auto HD_ACTIONDATA_ADR = ::IXBar::HD_ACTIONDATA_ADR;
188 static constexpr auto HD_PRECOLOR = ::IXBar::HD_PRECOLOR;
189 static constexpr auto HD_HASHMOD = ::IXBar::HD_HASHMOD;
190 static constexpr auto HD_DESTS = ::IXBar::HD_DESTS;
191
192 struct Use : public ::IXBar::Use {
193 bool gw_search_bus = false;
194 int gw_search_bus_bytes = 0;
195 bool gw_hash_group = false;
196 parity_status_t parity = PARITY_NONE;
197
198 bool search_data() const { return gw_search_bus || gw_hash_group; }
199 bool is_parity_enabled() const override { return parity == PARITY_ENABLED; }
200
201 HashDistDest_t hash_dist_type = HD_DESTS;
202 std::string hash_dist_used_for() const override {
203 return IXBar::hash_dist_name(hash_dist_type); }
204
205 /* which of the 16 hash tables we are using (bitvec) */
206 unsigned hash_table_inputs[HASH_GROUPS] = { 0 };
207 /* hash seed for different hash groups */
208 bitvec hash_seed[HASH_GROUPS];
209
210 /* tracking bits that are placed into the upper 12 bits of a hash group
211 * (with an identity hash) for use by a gateway */
212 struct Bits {
213 cstring field;
214 int group;
215 int lo, bit, width;
216 bool valid;
217 Bits(cstring f, int g, int l, int b, int w)
218 : field(f), group(g), lo(l), bit(b), width(w), valid(false) {}
219 Bits(cstring f, int g, int l, int b, int w, bool v)
220 : field(f), group(g), lo(l), bit(b), width(w), valid(v) {}
221 int hi() const { return lo + width - 1; } };
222 safe_vector<Bits> bit_use;
223
224 /* tracks hash use for Stateful and Selectors (and meter?) */
225 struct MeterAluHash {
226 bool allocated = false;
227 int group = -1;
228 bitvec bit_mask;
229 IR::MAU::HashFunction algorithm;
230 std::map<int, const IR::Expression *> computed_expressions;
231
232 void clear() {
233 allocated = false;
234 group = -1;
235 bit_mask.clear();
236 // identity_positions.clear();
237 computed_expressions.clear();
238 }
239 } meter_alu_hash;
240 const std::map<int, const IR::Expression *> &hash_computed_expressions() const override {
241 return meter_alu_hash.computed_expressions; }
242
243 /* tracks hash dist use (and hashes */
244 struct HashDistHash {
245 bool allocated = false;
246 int group = -1;
247 bitvec galois_matrix_bits;
248 IR::MAU::HashFunction algorithm;
249 std::map<int, le_bitrange> galois_start_bit_to_p4_hash;
250 const IR::Expression *hash_gen_expr = nullptr; // expression from HashGenExpr
251 cstring name; // Original name in case of hash distribution unit sharing
252
253 void clear() {
254 allocated = false;
255 group = -1;
256 galois_matrix_bits.clear();
257 galois_start_bit_to_p4_hash.clear();
258 }
259 } hash_dist_hash;
260 int hash_dist_hash_group() const override { return hash_dist_hash.group; }
261
262 struct ProxyHashKey {
263 bool allocated = false;
264 int group = -1;
265 bitvec hash_bits;
266 IR::MAU::HashFunction algorithm;
267
268 void clear() {
269 allocated = false;
270 group = -1;
271 hash_bits.clear();
272 }
273 } proxy_hash_key_use;
274
275 struct SaluInputSource {
276 unsigned data_bytemask = 0;
277 unsigned hash_bytemask = 0; // redundant with meter_alu_hash.bit_mask?
278
279 void clear() { data_bytemask = hash_bytemask = 0; }
280 bool empty() const { return !data_bytemask && !hash_bytemask; }
281 } salu_input_source;
282
283 // The order in the P4 program that the fields appear in the list
284 safe_vector<const IR::Expression *> field_list_order;
285 LTBitMatrix symmetric_keys;
286
287 void clear() override {
288 ::IXBar::Use::clear();
289 gw_search_bus = false;
290 gw_search_bus_bytes = 0;
291 gw_hash_group = false;
292 parity = PARITY_NONE;
293 hash_dist_type = HD_DESTS;
294 for (auto &ht : hash_table_inputs) ht = 0;
295 for (auto &hs : hash_seed) hs.clear();
296 bit_use.clear();
297 meter_alu_hash.clear();
298 hash_dist_hash.clear();
299 proxy_hash_key_use.clear();
300 salu_input_source.clear();
301 field_list_order.clear();
302 }
303 Use *clone() const override { return new Use(*this); }
304 bool empty() const override {
305 return ::IXBar::Use::empty() && bit_use.empty() &&
306 !meter_alu_hash.allocated && !hash_dist_hash.allocated &&
307 !proxy_hash_key_use.allocated && salu_input_source.empty(); }
308 void dbprint(std::ostream &) const override;
309
310 void add(const Use &alloc);
311 safe_vector<Byte> atcam_partition(int *hash_group = nullptr) const override;
312 safe_vector<TotalInfo> bits_per_search_bus() const override;
313 unsigned compute_hash_tables() override;
314 bool emit_gateway_asm(const MauAsmOutput &, std::ostream &, indent_t,
315 const IR::MAU::Table *) const override { return false; }
316 void emit_ixbar_asm(const PhvInfo &phv, std::ostream& out, indent_t indent,
317 const TableMatch *fmt, const IR::MAU::Table *) const override;
318 void emit_salu_bytemasks(std::ostream &out, indent_t indent) const override;
319 void emit_ixbar_hash_table(int hash_table, safe_vector<Slice> &match_data,
320 safe_vector<Slice> &ghost, const TableMatch *fmt,
321 std::map<int, std::map<int, Slice>> &sort) const override;
322 bitvec galois_matrix_bits() const override { return hash_dist_hash.galois_matrix_bits; }
323 int hash_groups() const override;
324 TotalBytes match_hash(safe_vector<int> *hash_groups = nullptr) const override;
325 bitvec meter_bit_mask() const override { return meter_alu_hash.bit_mask; }
326 int ternary_align(const Loc &) const override;
327 int total_input_bits() const override {
328 int rv = 0;
329 for (auto fl : field_list_order) {
330 rv += fl->type->width_bits(); }
331 return rv; }
332 void update_resources(int, BFN::Resources::StageResources &) const override;
333 const char *way_source_kind() const override { return "group"; }
334 };
335 static Use &getUse(autoclone_ptr<::IXBar::Use> &ac);
336 static const Use &getUse(const autoclone_ptr<::IXBar::Use> &ac);
337
347 struct HashDistAllocPostExpand {
348 P4HashFunction *func;
349 le_bitrange bits_in_use;
350 HashDistDest_t dest;
351 int shift;
352 // Only currently used for dynamic hash. Goal is to remove
353 const IR::MAU::HashDist *created_hd;
354 // Workaround for multi-stage fifo tests. Goal is to remove this as well and have
355 // the hash/compiler to generate this individually and determine it, but that's not
356 // very optimal in the given structure
357 bool chained_addr = false;
358 bitvec possible_shifts() const;
359
360 public:
361 HashDistAllocPostExpand(P4HashFunction *f, le_bitrange b, HashDistDest_t d, int s)
362 : func(f), bits_in_use(b), dest(d), shift(s) {}
363 bool operator<(const HashDistAllocPostExpand& hd) const {
364 return std::tie(dest, shift, bits_in_use, chained_addr) <
365 std::tie(hd.dest, hd.shift, hd.bits_in_use, hd.chained_addr);
366 }
367 };
368
369 struct HashDistIRUse {
370 autoclone_ptr<::IXBar::Use> use;
371 le_bitrange p4_hash_range;
372 HashDistDest_t dest;
373 // Only currently used for dynamic hash. Goal is to remove
374 const IR::MAU::HashDist *created_hd = nullptr;
375 cstring dyn_hash_name;
376 bool is_dynamic() const { return !dyn_hash_name.isNull(); }
377 };
378
379 struct HashDistUse {
380 // Source of this translated HashDistUse.
381 safe_vector<HashDistAllocPostExpand> src_reqs;
382
383 safe_vector<HashDistIRUse> ir_allocations;
384 int expand = -1;
385 int unit = -1;
386 int shift = -1;
387 bitvec mask;
388
389 std::set<cstring> outputs;
390
391 int hash_group() const;
392 bitvec destinations() const;
393 bitvec galois_matrix_bits() const;
394
395 cstring used_by;
396 std::string used_for() const;
397
398 void clear() {
399 src_reqs.clear();
400 ir_allocations.clear();
401 expand = -1;
402 unit = -1;
403 shift = 0;
404 mask.clear();
405 outputs.clear();
406 }
407 };
408
409 private:
410 ordered_map<const IR::MAU::Table *, const safe_vector<HashDistUse> *> tbl_hash_dists;
411 static unsigned hash_table_inputs(const HashDistUse &hdu);
412
413 class XBarHashDist : public MauInspector {
414 safe_vector<HashDistAllocPostExpand> alloc_reqs;
415 IXBar &self;
416 const PhvInfo &phv;
417 const IR::MAU::Table *tbl;
418 // const ActionFormat::Use *af;
419 const ActionData::Format::Use *af;
420 const LayoutOption *lo;
421 TableResourceAlloc *resources;
422 const attached_entries_t &attached_entries;
423
424 void build_action_data_req(const IR::MAU::HashDist *hd);
425 void build_req(const IR::MAU::HashDist *hd, const IR::Node *rel_node,
426 bool created_hd = false);
427
428 void build_function(const IR::MAU::HashDist *hd, P4HashFunction **func,
429 le_bitrange *bits = nullptr);
430 bool preorder(const IR::MAU::HashDist *) override;
431 bool preorder(const IR::MAU::TableSeq *) override { return false; }
432 bool preorder(const IR::MAU::AttachedOutput *) override { return false; }
433 bool preorder(const IR::MAU::StatefulCall *) override { return false; }
434
435 public:
436 void immediate_inputs();
437 void hash_action();
438 bool allocate_hash_dist();
439 XBarHashDist(IXBar &s, const PhvInfo &p, const IR::MAU::Table *t,
440 const ActionData::Format::Use *a, const LayoutOption *l, TableResourceAlloc *r,
441 const attached_entries_t &ae)
442 : self(s), phv(p), tbl(t), af(a), lo(l), resources(r), attached_entries(ae) {}
443 };
444
445 private:
446 enum AvailBytesPerRepeatingSect_t { AV_FULL = 1, AV_HALF = 2, AV_BYTE = 4 };
447
448/* An individual SRAM group or half of a TCAM group */
449 struct grp_use : public IHasDbPrint {
450 enum type_t { MATCH, HASH_DIST, FREE };
451 int group;
457 bitvec found;
458
472 std::array<bitvec, REPEATING_CONSTRAINT_SECT> _free = { { bitvec() } };
473 bitvec free(bitvec can_use) const {
474 bitvec rv;
475 for (auto bit : can_use)
476 rv |= _free.at(bit);
477 return rv;
478 }
479
480 bitvec max_free() const { return free(bitvec(0, REPEATING_CONSTRAINT_SECT)); }
481 bool attempted = false;
482 bool hash_open[2] = { true, true };
483 type_t hash_table_type[2] = { FREE, FREE };
484 int range_index = -1;
485
486 bool hash_dist_avail(int ht) const {
487 return hash_table_type[ht] == HASH_DIST || hash_table_type[ht] == FREE;
488 }
489
490 bool hash_dist_only(int ht) const {
491 return hash_table_type[ht] == HASH_DIST;
492 }
493
494 void dbprint(std::ostream &out) const {
495 out << group << " found: 0x" << found << " free: 0x" << max_free();
496 }
497
498 int total_avail() const { return found.popcount() + max_free().popcount(); }
499 bool range_set() const { return range_index != -1; }
500
501 bool is_better_group(const grp_use &b, bool prefer_found,
502 int required_allocation_bytes, std::map<int, int> &constraints_to_reqs) const;
503 explicit grp_use(int g) : group(g) {}
504 };
505
506 using mid_byte_use = grp_use;
507
508 public: // for gtest only
509 struct hash_matrix_reqs {
510 // The max number of groups that can be used by this table. Required by gateways,
511 // and stateful/meter tables using the search bus
512 int max_search_buses = -1;
513 int index_groups = 0;
514 int select_bits = 0;
515 bool hash_dist = false;
516 bool requires_versioning = false;
517
518 static hash_matrix_reqs max(bool hd, bool ternary = false) {
519 hash_matrix_reqs rv;
520 if (ternary)
521 return rv;
522 rv.hash_dist = hd;
523 if (rv.hash_dist) {
524 rv.index_groups = HASH_DIST_SLICES;
525 } else {
526 rv.index_groups = HASH_INDEX_GROUPS;
527 rv.select_bits = IXBar::get_hash_single_bits();
528 }
529 return rv;
530 }
531
532 bool fit_requirements(bitvec hash_matrix_in_use) const;
533
534 hash_matrix_reqs() {}
535 hash_matrix_reqs(int ig, int sb, bool hd)
536 : index_groups(ig), select_bits(sb), hash_dist(hd) {}
537 };
538
539 private:
540 // Used to determine what phv fields need to be allocated on the input xbar for the
541 // stateful ALU to work. Private internal to allocStateful
542 class FindSaluSources : public MauInspector {
543 const PhvInfo &phv;
544 ContByteConversion &map_alloc;
545 safe_vector<const IR::Expression *> &field_list_order;
546 // Holds which bitranges of fields have been requested, and will not allocate
547 // if a bitrange has been requested multiple times
548 std::map<cstring, bitvec> fields_needed;
549 const IR::MAU::Table *tbl;
550
551 profile_t init_apply(const IR::Node *root) override;
552 bool preorder(const IR::MAU::StatefulAlu *) override;
553 bool preorder(const IR::MAU::SaluAction *) override;
554 bool preorder(const IR::Expression *e) override;
555 bool preorder(const IR::MAU::HashDist *) override;
556 bool preorder(const IR::MAU::IXBarExpression *) override;
558 bool preorder(const IR::Annotation *) override { return false; }
559 bool preorder(const IR::Declaration_Instance *) override { return false; }
560 bool preorder(const IR::MAU::Selector *) override { return false; }
561
562 static void collapse_contained(std::map<le_bitrange, const IR::Expression *> &m);
563
564 public:
565 FindSaluSources(IXBar &, const PhvInfo &phv, ContByteConversion &ma,
566 safe_vector<const IR::Expression *> &flo, const IR::MAU::Table *t)
567 : phv(phv), map_alloc(ma), field_list_order(flo), tbl(t) {}
568
569 ordered_map<const PHV::Field *, std::map<le_bitrange, const IR::Expression *>>
570 phv_sources;
571 std::vector<const IR::MAU::IXBarExpression *> hash_sources;
572 bool dleft = false;
573 };
574
575 void clear();
576 bool allocMatch(bool ternary, const IR::MAU::Table *tbl, const PhvInfo &phv, Use &alloc,
577 safe_vector<IXBar::Use::Byte *> &alloced, hash_matrix_reqs &hm_reqs);
578 bool allocPartition(const IR::MAU::Table *tbl, const PhvInfo &phv, Use &alloc,
579 bool second_try);
580 int getHashGroup(unsigned hash_table_input, const hash_matrix_reqs *hm_reqs = nullptr);
581 void getHashDistGroups(unsigned hash_table_input, int hash_group_opt[HASH_DIST_UNITS]);
582 bool allocProxyHash(const IR::MAU::Table *tbl, const PhvInfo &phv,
583 const LayoutOption *lo, Use &alloc, Use &proxy_hash_alloc);
584 bool allocProxyHashKey(const IR::MAU::Table *tbl, const PhvInfo &phv, const LayoutOption *lo,
585 Use &proxy_hash_alloc, hash_matrix_reqs &hm_reqs);
586
587 bool allocAllHashWays(bool ternary, const IR::MAU::Table *tbl, Use &alloc,
588 const LayoutOption *layout_option, size_t start, size_t last,
589 const hash_matrix_reqs &hm_reqs);
590 bool allocHashWay(const IR::MAU::Table *tbl, const LayoutOption *layout_option,
591 size_t index, std::map<int, bitvec> &slice_to_select_bits, Use &alloc,
592 unsigned local_hash_table_input, unsigned hf_hash_table_input, int hash_group);
593 bool allocGateway(const IR::MAU::Table *, const PhvInfo &phv, Use &alloc,
594 const LayoutOption *lo, bool second_try);
595 bool allocSelector(const IR::MAU::Selector *, const IR::MAU::Table *, const PhvInfo &phv,
596 Use &alloc, cstring name);
597 bool allocStateful(const IR::MAU::StatefulAlu *, const IR::MAU::Table *, const PhvInfo &phv,
598 Use &alloc, bool on_search_bus);
599 bool allocMeter(const IR::MAU::Meter *, const IR::MAU::Table *, const PhvInfo &phv,
600 Use &alloc, bool on_search_bus);
601
602 bool allocTable(const IR::MAU::Table *tbl, const PhvInfo &phv, TableResourceAlloc &alloc,
603 const LayoutOption *lo, const ActionData::Format::Use *af,
604 const attached_entries_t &ae);
605 bool allocTable(const IR::MAU::Table *tbl, const IR::MAU::Table *gw, const PhvInfo &phv,
606 TableResourceAlloc &alloc, const LayoutOption *lo,
607 const ActionData::Format::Use *af, const attached_entries_t &ae) override;
608
609 void update(cstring name, const ::IXBar::Use &alloc) override;
610 void update(cstring name, const HashDistUse &hash_dist_alloc);
611 void update(const IR::MAU::Table *tbl, const TableResourceAlloc *rsrc) override;
612 void update(const IR::MAU::Table *tbl) override;
613 void add_collisions() override;
614 void verify_hash_matrix() const override;
615 void dbprint(std::ostream &) const override;
616
617 const Loc *findExactByte(PHV::Container c, int byte) const {
618 for (auto &p : Values(exact_fields.equal_range(c)))
619 if (exact_use.at(p.group, p.byte).second/8 == byte)
620 return &p;
621 /* FIXME -- what if it's in more than one place? */
622 return nullptr; }
623 unsigned find_balanced_group(Use &alloc, int way_size);
624
625 public: // for gtest
626 bool find_alloc(safe_vector<IXBar::Use::Byte> &alloc_use, bool ternary,
627 safe_vector<IXBar::Use::Byte *> &alloced, hash_matrix_reqs &hm_reqs,
628 unsigned byte_mask = ~0U);
629
630 private:
631 bitvec can_use_from_flags(int flags) const;
632 int groups(bool ternary) const;
633 int mid_bytes(bool ternary) const;
634 int bytes_per_group(bool ternary) const;
635
636 void increase_ternary_ixbar_space(int &groups_needed, int &nibbles_needed,
637 bool requires_versioning);
638 bool calculate_sizes(safe_vector<Use::Byte> &alloc_use, bool ternary, int &total_bytes_needed,
639 int &groups_needed, int &mid_bytes_needed, bool requires_versioning);
640 void initialize_orders(safe_vector<grp_use> &order, safe_vector<mid_byte_use> &mid_byte_order,
641 bool ternary);
642 void setup_byte_vectors(safe_vector<Use::Byte> &alloc_use, bool ternary,
643 safe_vector<Use::Byte *> &unalloced, safe_vector<Use::Byte *> &alloced,
644 safe_vector<grp_use> &order, safe_vector<mid_byte_use> &mid_byte_order,
645 hash_matrix_reqs &hm_reqs, unsigned byte_mask);
646
647 void print_groups(safe_vector<grp_use> &order, safe_vector<mid_byte_use> &mid_byte_order,
648 bool ternary);
649 void fill_out_use(safe_vector<IXBar::Use::Byte *> &alloced, bool ternary);
650 void calculate_available_groups(safe_vector<grp_use> &order, hash_matrix_reqs &hm_reqs);
651 void calculate_available_hash_dist_groups(safe_vector<grp_use> &order,
652 hash_matrix_reqs &hm_reqs);
653 grp_use::type_t is_group_for_hash_dist(int hash_table);
654 bool violates_hash_constraints(safe_vector <grp_use> &order, bool hash_dist, int group,
655 int byte);
656 void reset_orders(safe_vector<grp_use> &order, safe_vector<mid_byte_use> &mid_byte_order);
657 void calculate_found(safe_vector<Use::Byte *> &unalloced, safe_vector<grp_use> &order,
658 safe_vector<mid_byte_use> &mid_byte_order, bool ternary, bool hash_dist,
659 unsigned byte_mask);
660 void calculate_free(safe_vector<grp_use> &order, safe_vector<mid_byte_use> &mid_byte_order,
661 bool ternary, bool hash_dist, unsigned byte_mask);
662 void found_bytes(grp_use *grp, safe_vector<IXBar::Use::Byte *> &unalloced, bool ternary,
663 int &match_bytes_placed, int search_bus);
664 void found_mid_bytes(mid_byte_use *mb_grp, safe_vector<Use::Byte *> &unalloced,
665 bool ternary, int &match_bytes_placed, int search_bus, bool &prefer_half_bytes,
666 bool only_alloc_nibble);
667 void free_bytes(grp_use *grp, safe_vector<Use::Byte *> &unalloced,
668 safe_vector<Use::Byte *> &alloced, bool ternary, bool hash_dist, int &match_bytes_placed,
669 int search_bus);
670 void free_mid_bytes(mid_byte_use *mb_grp, safe_vector<Use::Byte *> &unalloced,
671 safe_vector<Use::Byte *> &alloced, int &match_bytes_placed, int search_bus,
672 bool &prefer_half_bytes, bool only_alloc_nibble);
673 void allocate_byte(bitvec *bv, safe_vector<IXBar::Use::Byte *> &unalloced,
674 safe_vector<IXBar::Use::Byte *> *alloced, IXBar::Use::Byte &need, int group,
675 int byte, size_t &index, int &free_bytes, int &ixbar_bytes_placed,
676 int &match_bytes_placed, int search_bus, int *range_index);
677 void allocate_mid_bytes(safe_vector<Use::Byte *> &unalloced,
678 safe_vector<Use::Byte *> &alloced, bool ternary, bool prefer_found,
679 safe_vector<grp_use> &order, safe_vector<mid_byte_use> &mid_byte_order,
680 int nibbles_needed, int &bytes_to_allocate);
681
682 int determine_best_group(safe_vector<Use::Byte *> &unalloced, safe_vector<grp_use> &order,
683 bool ternary, bool prefer_found, int required_allocation_bytes);
684
685 void allocate_groups(safe_vector<Use::Byte *> &unalloced, safe_vector<Use::Byte *> &alloced,
686 bool ternary, bool prefer_found, safe_vector<grp_use> &order,
687 safe_vector<mid_byte_use> &mid_byte_order, int &bytes_to_allocate,
688 int groups_needed, bool hash_dist, unsigned byte_mask);
689 bool handle_pragma_ixbar_group_num(safe_vector<Use::Byte *> &unalloced,
690 safe_vector<Use::Byte *> &alloced, bool ternary,
691 safe_vector<grp_use> &order,
692 safe_vector<mid_byte_use> &mid_byte_order,
693 int &bytes_to_allocate, bool hash_dist,
694 unsigned byte_mask, int search_bus);
695
696 hash_matrix_reqs match_hash_reqs(const LayoutOption *lo, size_t start, size_t end,
697 bool ternary);
698 void layout_option_calculation(const LayoutOption *layout_option,
699 size_t &start, size_t &last);
700
701 int max_bit_to_byte(bitvec bit_mask);
702 int max_index_group(int max_bit);
703 int max_index_single_bit(int max_bit);
704 unsigned index_groups_used(bitvec bv) const;
705 unsigned select_bits_used(bitvec bv) const;
706 bool hash_use_free(int max_group, int max_single_bit, unsigned hash_table_input);
707 void write_hash_use(int max_group, int max_single_bit, unsigned hash_table_input,
708 cstring name);
709 bool can_allocate_on_search_bus(Use &alloc, const PHV::Field *field, le_bitrange range,
710 int ixbar_position);
711 bool setup_stateful_search_bus(const IR::MAU::StatefulAlu *salu, Use &alloc,
712 const FindSaluSources &sources, unsigned &byte_mask);
713 bool setup_stateful_hash_bus(const PhvInfo &, const IR::MAU::StatefulAlu *salu, Use &alloc,
714 const FindSaluSources &sources);
715
716 bitvec determine_final_xor(const IR::MAU::HashFunction *hf, const PhvInfo &phv,
717 std::map<int, le_bitrange> &bit_starts,
718 safe_vector<const IR::Expression *> field_list, int total_input_bits);
719 void determine_proxy_hash_alg(const PhvInfo &, const IR::MAU::Table *tbl, Use &use, int group);
720
721 bool isHashDistAddress(HashDistDest_t dest) const;
722
723 void determineHashDistInUse(int hash_group, bitvec &units_in_use, bitvec &hash_bits_in_use);
724 void buildHashDistIRUse(HashDistAllocPostExpand &alloc_req, HashDistUse &use,
725 IXBar::Use &all_reqs, const PhvInfo &phv, int hash_group, bitvec hash_bits_used,
726 bitvec total_post_expand_bits, const IR::MAU::Table* tbl, cstring name);
727 void lockInHashDistArrays(safe_vector<Use::Byte *> *alloced, int hash_group,
728 unsigned hash_table_input, int asm_unit, bitvec hash_bits_used, HashDistDest_t dest,
729 cstring name);
730
731
732 bool allocHashDistSection(bitvec post_expand_bits, bitvec possible_shifts, int hash_group,
733 int &unit_allocated, bitvec &hash_bits_allocated);
734 bool allocHashDistWideAddress(bitvec post_expand_bits, bitvec possible_shifts, int hash_group,
735 bool chained_addr, int &unit_allocated, bitvec &hash_bits_allocated);
736 bool allocHashDist(safe_vector<HashDistAllocPostExpand> &alloc_reqs, HashDistUse &use,
737 const PhvInfo &phv, cstring name, const IR::MAU::Table* tbl, bool second_try);
738 void createChainedHashDist(const HashDistUse &hd_alloc, HashDistUse &chained_hd_alloc,
739 cstring name);
740 void update_hash_parity(IXBar::Use &use, int hash_group);
741 parity_status_t update_hash_parity(int hash_group);
742};
743
744} // namespace Tofino
745
746/* clang-format on */
747
748#endif /* BF_P4C_MAU_TOFINO_INPUT_XBAR_H_ */
BFN::Alloc1D
Definition alloc.h:65
BFN::Alloc2D
Definition alloc.h:152
BFN::Alloc2Dbase
Definition alloc.h:76
LayoutOption
Definition table_layout.h:34
MauAsmOutput
Definition mau/asm_output.h:85
MauInspector
Definition mau_visitor.h:29
P4::IHasDbPrint
Definition stringify.h:33
P4::IR::Node
Definition node.h:94
P4::LTBitMatrix
Definition ltbitmatrix.h:25
P4::bitvec
Definition bitvec.h:120
P4::cstring
Definition cstring.h:85
P4::indent_t
Definition indent.h:26
P4::ordered_map
Definition ordered_map.h:32
P4::safe_vector
Definition safe_vector.h:27
PHV::Container
Definition phv.h:176
PHV::Field
Definition phv_fields.h:154
PhvInfo
Definition phv_fields.h:1095
TableMatch
Definition mau/asm_output.h:223
autoclone_ptr
Definition autoclone.h:31
P4
TODO: this is not really specific to BMV2, it should reside somewhere else.
Definition applyOptionsPragmas.cpp:24
Tofino
Definition tofino/action_data_bus.cpp:28
ActionData::Format::Use
Definition action_format.h:979
BFN::Resources::StageResources
Definition resources.h:125
IXBar::FieldInfo
Definition input_xbar.h:128
IXBar::Loc
Definition input_xbar.h:74
IXBar::Use::Byte
Definition input_xbar.h:230
IXBar::Use
Definition input_xbar.h:191
IXBar
Definition input_xbar.h:69
IXBar::ContByteConversion
std::map< Use::Byte, safe_vector< FieldInfo > > ContByteConversion
Definition input_xbar.h:434
P4::ClosedRange::lo
int lo
Definition lib/bitrange.h:694
P4::IR::MAU::HashFunction
Definition hash_function.h:28
P4HashFunction
Definition ixbar_expr.h:209
TableResourceAlloc
Definition resource.h:37
Tofino::IXBar::HashDistAllocPostExpand
Definition tofino/input_xbar.h:347
Tofino::IXBar::HashDistIRUse
Definition tofino/input_xbar.h:369
Tofino::IXBar::HashDistUse
Definition tofino/input_xbar.h:379
Tofino::IXBar::Use::Bits
Definition tofino/input_xbar.h:212
Tofino::IXBar::Use::HashDistHash
Definition tofino/input_xbar.h:244
Tofino::IXBar::Use::MeterAluHash
Definition tofino/input_xbar.h:225
Tofino::IXBar::Use::ProxyHashKey
Definition tofino/input_xbar.h:262
Tofino::IXBar::Use::SaluInputSource
Definition tofino/input_xbar.h:275
Tofino::IXBar::Use
Definition tofino/input_xbar.h:192
Tofino::IXBar::Use::match_hash
TotalBytes match_hash(safe_vector< int > *hash_groups=nullptr) const override
Definition mau/tofino/input_xbar.cpp:264
Tofino::IXBar::Use::bits_per_search_bus
safe_vector< TotalInfo > bits_per_search_bus() const override
Definition mau/tofino/input_xbar.cpp:150
Tofino::IXBar::Use::atcam_partition
safe_vector< Byte > atcam_partition(int *hash_group=nullptr) const override
Definition mau/tofino/input_xbar.cpp:134
Tofino::IXBar::hash_matrix_reqs
Definition tofino/input_xbar.h:509
Tofino::IXBar
Definition tofino/input_xbar.h:30
Tofino::IXBar::find_alloc
bool find_alloc(safe_vector< IXBar::Use::Byte > &alloc_use, bool ternary, safe_vector< IXBar::Use::Byte * > &alloced, hash_matrix_reqs &hm_reqs, unsigned byte_mask=~0U)
Definition mau/tofino/input_xbar.cpp:1347
Tofino::IXBar::SELECT_BIT_RANGE
static constexpr le_bitrange SELECT_BIT_RANGE
Definition tofino/input_xbar.h:61