source file
rtl/datamover_engine.sv
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 | /* * Copyright (C) 2025-2026 ETH Zurich and University of Bologna * * Copyright and related rights are licensed under the Solderpad Hardware * License, Version 0.51 (the "License"); you may not use this file except in * compliance with the License. You may obtain a copy of the License at * http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law * or agreed to in writing, software, hardware and materials distributed under * this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ /* * Authors: Francesco Conti <f.conti@unibo.it> * Sergio Mazzola <smazzola@iis.ee.ethz.ch> * Cyrill Durrer <cdurrer@iis.ee.ethz.ch> */ `include "common_cells/registers.svh" module datamover_engine import hwpe_stream_package::*; import hci_package::*; import datamover_package::*; #( parameter int unsigned FIFO_DEPTH = 2, parameter int unsigned BANDWIDTH_ALIGNED = 512, parameter int unsigned NUM_ELEM_WORD = 4, // number of elements in a bank word parameter int unsigned ELEM_WIDTH = 8, // element width (in bits) // Dependent parameters: do not modify! localparam int unsigned WORD_WIDTH = NUM_ELEM_WORD * ELEM_WIDTH // should correspond to bank width ) ( // global signals input logic clk_i, input logic rst_ni, input logic test_mode_i, // unused // local enable & clear input logic enable_i, // unused input logic clear_i, // control registers input ctrl_engine_t ctrl_i, // input data stream + handshake hwpe_stream_intf_stream.sink data_in, // output data stream + handshake hwpe_stream_intf_stream.source data_out ); // number of elements (in the full bandwidth, not a single bank word) localparam int unsigned NB_ELEMENTS = BANDWIDTH_ALIGNED / ELEM_WIDTH; localparam int unsigned NB_ELEM_LOG2 = $clog2(NB_ELEMENTS); // Counter widths // TILE_CNT holds ceil(tensor_size / NB_ELEMENTS) // ELEM_CNT holds tensor_size rounded up to a full tile // ACCESS_CNT holds the output beats of one job localparam int unsigned TILE_CNT_WIDTH = TENSOR_SIZE_WIDTH - NB_ELEM_LOG2 + 1; localparam int unsigned ELEM_CNT_WIDTH = TENSOR_SIZE_WIDTH + 1; localparam int unsigned ACCESS_CNT_BASE = (TENSOR_SIZE_WIDTH + NB_ELEM_LOG2 > TOTAL_ELEM_WIDTH) ? TENSOR_SIZE_WIDTH + NB_ELEM_LOG2 : TOTAL_ELEM_WIDTH; localparam int unsigned ACCESS_CNT_WIDTH = ACCESS_CNT_BASE + 2; // Type def and internal signals typedef enum logic { WRITE, READ } datamover_engine_fsm_t; datamover_engine_fsm_t fsm_d, fsm_q; logic clear_elem_matrix; logic clear_run; logic [NB_ELEM_LOG2-1:0] cnt_q, cnt_d; logic [ACCESS_CNT_WIDTH-1:0] tot_cnt_q, tot_cnt_d; logic cnt_en; logic tot_cnt_incr; logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_in_unrolled; logic data_in_valid; logic data_in_ready; logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_out_unrolled; logic data_out_valid; logic data_out_ready; logic [NB_ELEM_LOG2-1:0] remaining_elems; logic [ACCESS_CNT_WIDTH-1:0] total_accesses_copy_mode, total_accesses, acc_target; logic [ELEM_CNT_WIDTH-1:0] y_elem_cnt_d, y_elem_cnt_q, expanded_y_elems; logic y_elem_wrap; logic [TILE_CNT_WIDTH-1:0] y_tiles, n_tiles, n_tile_cnt_d, n_tile_cnt_q; logic [NB_ELEM_LOG2:0] leftover_rows, leftover_cols; logic last_y_tile, last_n_tile; logic [TILE_CNT_WIDTH-1:0] tile_y_q, tile_y_d, tile_n_q, tile_n_d; logic [NB_ELEM_LOG2:0] write_len, read_len, phase_len; logic tile_complete, transpose_done, tp_last_y, tp_last_n; logic execution_done; logic im2col_pack, im2col_pad; logic pack_wr_lo, pack_half_q; logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] pack_extract; logic [NB_ELEMENTS-1:0] pad_zero; assign im2col_pack = ctrl_i.im2col_pack; assign im2col_pad = ctrl_i.im2col_pad; // FSM: WRITE -> READ on input handshake at end of write, READ -> WRITE on output handshake at end of read always_comb begin fsm_d = fsm_q; case (fsm_q) WRITE: begin if (((cnt_q == phase_len-ctrl_i.transp_stride)) && (data_in_valid & data_in_ready)) begin fsm_d = READ; end end READ: begin if (((cnt_q == phase_len-ctrl_i.transp_stride)) && (data_out_valid & data_out_ready)) begin fsm_d = WRITE; end end default: begin fsm_d = WRITE; end endcase end assign clear_elem_matrix = (fsm_q == READ && fsm_d == WRITE) && (ctrl_i.transp_mode != TRANSP_NONE); assign clear_run = clear_i || execution_done; // internal interfaces and unrolling hwpe_stream_intf_stream #( .DATA_WIDTH ( BANDWIDTH_ALIGNED ), .ELEMENT_WIDTH ( ELEM_WIDTH ), .STRB_WIDTH ( NB_ELEMENTS ) ) data_in_postfifo ( .clk ( clk_i ) ); hwpe_stream_intf_stream #( .DATA_WIDTH ( BANDWIDTH_ALIGNED ), .ELEMENT_WIDTH ( ELEM_WIDTH ), .STRB_WIDTH ( NB_ELEMENTS ) ) data_out_prefifo ( .clk ( clk_i ) ); // decouple in/out with FIFOs hwpe_stream_fifo #( .DATA_WIDTH ( BANDWIDTH_ALIGNED ), .ELEMENT_WIDTH ( ELEM_WIDTH ), .FIFO_DEPTH ( FIFO_DEPTH ) ) i_fifo_in ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), .clear_i ( clear_i ), .flags_o ( ), .push_i ( data_in ), .pop_o ( data_in_postfifo ) ); assign data_in_unrolled = data_in_postfifo.data; assign data_in_valid = data_in_postfifo.valid; assign data_in_postfifo.ready = data_in_ready; hwpe_stream_fifo #( .DATA_WIDTH ( BANDWIDTH_ALIGNED ), .ELEMENT_WIDTH ( ELEM_WIDTH ), .FIFO_DEPTH ( FIFO_DEPTH ) ) i_fifo_out ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), .clear_i ( clear_i ), .flags_o ( ), .push_i ( data_out_prefifo ), .pop_o ( data_out ) ); // Partial tile / leftover elements handling // Due to the streamer address generation, matrices need to be word-aligned in n-dimension for transposition localparam logic [NB_ELEMENTS-1:0] STRB_ONE = {{(NB_ELEMENTS-1){1'b0}}, 1'b1}; // Necessary to force the shifting operation to the correct bitwidth (default would be only 32b) assign remaining_elems = ctrl_i.total_elements & (NB_ELEMENTS - 1); // modulo (NB_ELEMENTS: power of two) - this signal is only used in copy mode assign total_accesses_copy_mode = (ctrl_i.total_elements >> NB_ELEM_LOG2) + ((remaining_elems != 0) ? 1 : 0); // y_tiles represents the number of tiles in c-dimension for unfold/fold modes, and the number of tiles in m-dimension for all other modes assign y_tiles = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? (ctrl_i.num_channels + NB_ELEMENTS - 1) >> NB_ELEM_LOG2 : (ctrl_i.tensor_size_m + NB_ELEMENTS - 1) >> NB_ELEM_LOG2; // ceil division assign n_tiles = (ctrl_i.tensor_size_n + NB_ELEMENTS - 1) >> NB_ELEM_LOG2; // ceil division assign total_accesses = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? (y_tiles * ctrl_i.tensor_size_m * n_tiles) << NB_ELEM_LOG2 : (y_tiles * n_tiles) << NB_ELEM_LOG2; // NB_ELEMENTS is a power of 2, so multiply by shifting; ToDo: remaining MUL overhead, could be pre-computed in HAL and configured in control register assign leftover_rows = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? ctrl_i.num_channels & (NB_ELEMENTS - 1) : ctrl_i.tensor_size_m & (NB_ELEMENTS - 1); assign leftover_cols = ctrl_i.tensor_size_n & (NB_ELEMENTS - 1); assign expanded_y_elems = y_tiles << NB_ELEM_LOG2; // taking into account partial tiles assign last_y_tile = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? ((y_elem_cnt_q >> NB_ELEM_LOG2) >= (ctrl_i.num_channels >> NB_ELEM_LOG2)) : ((y_elem_cnt_q >> NB_ELEM_LOG2) >= (ctrl_i.tensor_size_m >> NB_ELEM_LOG2)); assign last_n_tile = (n_tile_cnt_q >= (ctrl_i.tensor_size_n >> NB_ELEM_LOG2)); // Partial-tile transpose gating assign tp_last_y = (tile_y_q == y_tiles - 1); assign tp_last_n = (tile_n_q == n_tiles - 1); assign write_len = ((ctrl_i.datamover_mode == DATAMOVER_TRANSPOSE) && tp_last_y && (leftover_rows != 0)) ? leftover_rows : ctrl_i.transp_len; assign read_len = ((ctrl_i.datamover_mode == DATAMOVER_TRANSPOSE) && tp_last_n && (leftover_cols != 0)) ? leftover_cols : ctrl_i.transp_len; assign phase_len = (fsm_q == WRITE) ? write_len : read_len; assign tile_complete = (ctrl_i.datamover_mode == DATAMOVER_TRANSPOSE) && clear_elem_matrix; assign tile_y_d = tile_complete ? (tp_last_y ? '0 : tile_y_q + 1'b1) : tile_y_q; assign tile_n_d = (tile_complete && tp_last_y) ? tile_n_q + 1'b1 : tile_n_q; logic [NB_ELEMENTS-1:0] strb_copy, strb_transpose, strb_unfold, strb_cim_fold, strb_im2col; assign strb_copy = ((tot_cnt_q >= total_accesses_copy_mode-1) && (remaining_elems != 0)) ? ((STRB_ONE << remaining_elems) - 1) : '1; // Transpose drains write_len valid rows of the last tile assign strb_transpose = (STRB_ONE << write_len) - 1; // Sub-BW im2col rows (tensor_size_n = W_out < NB_ELEMENTS); packed/padded stores are full. assign strb_im2col = (im2col_pack || im2col_pad) ? '1 : (ctrl_i.tensor_size_n < NB_ELEMENTS) ? ((STRB_ONE << ctrl_i.tensor_size_n) - 1) : strb_copy; assign strb_unfold = ((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) ? (((y_elem_cnt_q & (NB_ELEMENTS - 1)) < leftover_cols) ? ((STRB_ONE << leftover_rows) - 1) : '0) : (last_y_tile && leftover_rows != 0) ? ((STRB_ONE << leftover_rows) - 1) : (last_n_tile && leftover_cols != 0) ? (((y_elem_cnt_q & (NB_ELEMENTS - 1)) < leftover_cols) ? '1 : '0) : '1; assign strb_cim_fold = ((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) ? (((y_elem_cnt_q & (NB_ELEMENTS - 1)) < leftover_rows) ? ((STRB_ONE << leftover_cols) - 1) : '0) : (last_y_tile && leftover_rows != 0) ? (((y_elem_cnt_q & (NB_ELEMENTS - 1)) < leftover_rows) ? '1 : '0) : (last_n_tile && leftover_cols != 0) ? ((STRB_ONE << leftover_cols) - 1) : '1; assign data_out_prefifo.strb = (ctrl_i.total_elements == 0) ? '1 : (ctrl_i.datamover_mode == DATAMOVER_COPY) ? strb_copy : // Copy mode (ctrl_i.datamover_mode == DATAMOVER_IM2COL) ? strb_im2col : // Im2col mode (ctrl_i.datamover_mode == DATAMOVER_TRANSPOSE) ? strb_transpose : // Transpose mode (ctrl_i.datamover_mode == DATAMOVER_UNFOLD) ? strb_unfold : // Unfold mode (ctrl_i.datamover_mode == DATAMOVER_CIM_CONVERSION || // CIM layout conversion mode ctrl_i.datamover_mode == DATAMOVER_FOLD) ? strb_cim_fold : // Fold mode (inverse of unfold: leftover_rows <-> leftover_cols roles swapped) '1; assign data_out_prefifo.data = data_out_unrolled; assign data_out_prefifo.valid = data_out_valid; assign data_out_ready = data_out_prefifo.ready; // Write counter assign cnt_en = fsm_q == WRITE ? data_in_valid & data_in_ready : data_out_valid & data_out_ready; assign cnt_d = cnt_en ? ((cnt_q < (phase_len-ctrl_i.transp_stride)) ? cnt_q+ctrl_i.transp_stride : '0) : cnt_q; assign transpose_done = tp_last_y && tp_last_n && clear_elem_matrix; // COPY/IM2COL count against total_accesses_copy_mode; UNFOLD/FOLD/CIM against total_accesses. assign acc_target = ((ctrl_i.datamover_mode == DATAMOVER_COPY) || (ctrl_i.datamover_mode == DATAMOVER_IM2COL)) ? total_accesses_copy_mode : total_accesses; assign execution_done = (ctrl_i.datamover_mode == DATAMOVER_TRANSPOSE) ? transpose_done : (acc_target != 0) && (data_out_prefifo.valid & data_out_prefifo.ready) && (tot_cnt_q >= acc_target - 1); assign tot_cnt_incr = (im2col_pack ? 1'b1 : cnt_en) & (data_out_prefifo.valid & data_out_prefifo.ready); // count total number of write accesses assign tot_cnt_d = tot_cnt_incr ? tot_cnt_q + 1 : tot_cnt_q; assign y_elem_wrap = (y_elem_cnt_q == expanded_y_elems - 1); assign y_elem_cnt_d = tot_cnt_incr ? (y_elem_wrap ? '0 : y_elem_cnt_q + 1) : y_elem_cnt_q; assign n_tile_cnt_d = (tot_cnt_incr & y_elem_wrap) ? n_tile_cnt_q + 1 : n_tile_cnt_q; // "Smart shifting": this set of combinational blocks shifts data_in_unrolled // appropriately, depending on the configuration. // E.g., if you have a classical configuration with // - NUM_ELEM_WORD = 8 and // - ELEM_WIDTH = 8 bits, i.e. total is 64 bits per word // the configurations are: 8b transpose, 16b transpose, 32b transpose. We assume // that transposes >= 64b can be done efficiently by Snitch processors through SSRs // and those < 8b are not interesting in our use case. localparam MAX_SHIFTING = (NUM_ELEM_WORD > MAX_TRANSP_STRIDE) ? NUM_ELEM_WORD : MAX_TRANSP_STRIDE; // e.g., in a classical configuration (ELEM_WIDTH = 8), MAX_SHIFTING is // in bytes, i.e., "4" for 32b transpose (includes shifting by 0 bytes) logic [MAX_SHIFTING-1:0][NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_in_shifted; for(genvar ii=0; ii<MAX_SHIFTING; ii++) begin : gen_data_shifting_x for(genvar jj=0; jj<NB_ELEMENTS; jj++) begin : gen_data_shifting_y if(ii+jj < NB_ELEMENTS) begin : gen_feasible_shiftings assign data_in_shifted[ii][jj] = data_in_unrolled[ii+jj]; end else begin : gen_unfeasible_shiftings assign data_in_shifted[ii][jj] = '0; end end // gen_data_shifting_y end // gen_data_shifting_x datamover_im2col_ctrl #( .NB_ELEMENTS ( NB_ELEMENTS ), .ELEM_WIDTH ( ELEM_WIDTH ) ) i_im2col_ctrl ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), .clear_run_i ( clear_run ), .ctrl_i ( ctrl_i ), .data_in_unrolled_i ( data_in_unrolled ), .data_in_valid_i ( data_in_valid ), .data_in_ready_i ( data_in_ready ), .pack_extract_o ( pack_extract ), .pack_wr_lo_o ( pack_wr_lo ), .pack_half_q_o ( pack_half_q ), .pad_zero_o ( pad_zero ) ); logic [NB_ELEMENTS-1:0] wr_row_en; logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] wr_row_data [NB_ELEMENTS-1:0]; logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] elem_matrix_q [NB_ELEMENTS-1:0]; for(genvar ii=0; ii<NB_ELEMENTS; ii++) begin : gen_buffer_write logic in_hs, buffer_enable; assign in_hs = data_in_valid & data_in_ready; assign buffer_enable = ctrl_i.transp_mode == TRANSP_NONE ? 1'b0 : ctrl_i.transp_mode == TRANSP_4ELEM ? ((cnt_q>>2) == (ii>>2)) & in_hs : ctrl_i.transp_mode == TRANSP_2ELEM ? ((cnt_q>>1) == (ii>>1)) & in_hs : ( cnt_q == ii ) & in_hs; logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_in_selected; assign data_in_selected = ctrl_i.transp_mode == TRANSP_4ELEM? data_in_shifted[ii % 4] : ctrl_i.transp_mode == TRANSP_2ELEM ? data_in_shifted[ii % 2] : data_in_shifted[0]; assign wr_row_en[ii] = (im2col_pack && (ii == 0)) ? pack_wr_lo : buffer_enable; assign wr_row_data[ii] = (im2col_pack && (ii == 0)) ? pack_extract : data_in_selected; end // gen_buffer_write datamover_buffer #( .NB_ELEMENTS ( NB_ELEMENTS ), .ELEM_WIDTH ( ELEM_WIDTH ) ) i_buffer ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), .clear_i ( clear_i ), .clear_matrix_i ( clear_elem_matrix ), .wr_row_en_i ( wr_row_en ), .wr_row_data_i ( wr_row_data ), .rd_data_o ( elem_matrix_q ) ); // Output assignment for(genvar ii=0; ii<NB_ELEMENTS; ii++) begin : gen_output assign data_out_unrolled[ii] = ctrl_i.transp_mode != TRANSP_NONE ? elem_matrix_q[ii][cnt_q] : im2col_pack ? (ii < NB_ELEMENTS/2 ? elem_matrix_q[0][ii] : pack_extract[(ii+NB_ELEMENTS/2) % NB_ELEMENTS]) : (im2col_pad && pad_zero[ii]) ? '0 : data_in_unrolled[NB_ELEM_LOG2'(ii*ctrl_i.conv_stride)]; end // gen_output assign data_in_ready = ctrl_i.transp_mode != TRANSP_NONE ? fsm_q == WRITE : im2col_pack ? (pack_half_q == 1'b0 ? 1'b1 : data_out_ready) : data_out_ready; assign data_out_valid = ctrl_i.transp_mode != TRANSP_NONE ? fsm_q == READ : im2col_pack ? (pack_half_q == 1'b1 && data_in_valid) : data_in_valid; // Sequential logic `FFARNC(fsm_q, fsm_d, clear_run, WRITE, clk_i, rst_ni) `FFARNC(cnt_q, cnt_d, clear_run, '0, clk_i, rst_ni) `FFARNC(tot_cnt_q, tot_cnt_d, clear_run, '0, clk_i, rst_ni) `FFARNC(y_elem_cnt_q, y_elem_cnt_d, clear_run, '0, clk_i, rst_ni) `FFARNC(n_tile_cnt_q, n_tile_cnt_d, clear_run, '0, clk_i, rst_ni) `FFARNC(tile_y_q, tile_y_d, clear_run, '0, clk_i, rst_ni) `FFARNC(tile_n_q, tile_n_d, clear_run, '0, clk_i, rst_ni) `ifndef SYNTHESIS `ifndef VERILATOR `ifndef VCS // Parameter assertions (elaboration-time checks) initial begin assert (BANDWIDTH_ALIGNED <= MAX_BANDWIDTH) else $fatal("BANDWIDTH_ALIGNED (%0d) must not be greater than MAX_BANDWIDTH (%0d)", BANDWIDTH_ALIGNED, MAX_BANDWIDTH); assert ((BANDWIDTH_ALIGNED % WORD_WIDTH) == 0) else $fatal("BANDWIDTH_ALIGNED (%0d) must be a multiple of WORD_WIDTH (%0d)", BANDWIDTH_ALIGNED, WORD_WIDTH); assert ((NB_ELEMENTS != 0) && ((NB_ELEMENTS & (NB_ELEMENTS - 1)) == 0)) else $fatal("NB_ELEMENTS (%0d) = BANDWIDTH_ALIGNED (%0d) / ELEM_WIDTH (%0d) must be a power of two", NB_ELEMENTS, BANDWIDTH_ALIGNED, ELEM_WIDTH); assert (NUM_ELEM_WORD <= MAX_SHIFTING) else $fatal("NUM_ELEM_WORD (%0d) must not be greater than MAX_SHIFTING (%0d)", NUM_ELEM_WORD, MAX_SHIFTING); end // Runtime assertions assert property (@(posedge clk_i) disable iff (!rst_ni || $isunknown(ctrl_i.transp_len)) ctrl_i.transp_len <= NB_ELEMENTS ) else $error("transp_len (%0d) exceeds NB_ELEMENTS (%0d) - cnt_q will never match FSM transition condition", ctrl_i.transp_len, NB_ELEMENTS); `endif `endif `endif endmodule // datamover_engine |