add client throughput

app.py

@@ -138,8 +138,9 @@ with col3: # Prefilling
     prefilling_operation_count = prefilling_operation(model_config, inference_config)
     prefilling_activation_memory_count = prefilling_activation_memory(model_config, inference_config)
     inference_info['inference_prefilling_time'] = prefilling_operation_count['total'] / (gpu_config['TFLOP']*1024**4)
-    inference_info['inference_prefilling_throughput'] = inference_config['input_seq_length']*inference_config['batchsize']/inference_info['inference_prefilling_time']
     inference_info['prefilling_memory_latency'] = prefilling_activation_memory_count['total'] / (gpu_config['memory_bandwidth']*1024**3)
+    calc_prefilling_throughput(model_config, inference_config, inference_info)
+    
     cached_parameter_count['kv_cache'] = 2 * (inference_config['batchsize'] * (model_config['hidden_size'] * model_config['num_hidden_layers'] * inference_config['input_seq_length']))
 
     operation_items = {key: "{:,}".format(int(prefilling_operation_count[key])) for key in prefilling_operation_count if key not in subtotal_operations}
@@ -162,9 +163,9 @@ with col3: # Prefilling
 
     header5("Summary: Prefilling")
     st.markdown(create_table(df_subtotal_operation_count))
-    st.write(f"Prefillng throughput (tokens/s): {inference_info['inference_prefilling_throughput']:.2f}")
     st.write(f"FLOPS latency: {inference_info['inference_prefilling_time']}")
     st.write(f"Memory latency: {inference_info['prefilling_memory_latency']}")
+    st.write(f"Prefillng throughput (tokens/s): {inference_info['prefilling_throughput']:.2f} ({inference_info['prefilling_bound_type']}-bound)")
 
     if inference_config['KV_cache']:
         st.write(f"kv cache (Byte): {cached_parameter_count['kv_cache']:,}")
@@ -175,9 +176,9 @@ with col4: # Generation
     generation_operation_count = generation_operation(model_config, inference_config)
     generation_activation_memory_count = generation_activation_memory(model_config, inference_config)
     inference_info['inference_generation_time'] = generation_operation_count['total'] / (gpu_config['TFLOP']*1024**4)
-    inference_info['inference_generation_throughput'] = inference_config['output_seq_length']*inference_config['batchsize']/inference_info['inference_generation_time']
-    inference_info['inference_client_generation_throughput'] = inference_config['output_seq_length']*inference_config['batchsize'] / (inference_info['inference_prefilling_time'] + inference_info['inference_generation_time'])
     inference_info['generation_memory_latency'] = generation_activation_memory_count['total'] / (gpu_config['memory_bandwidth']*1024**3)
+    calc_generation_throughput(model_config, inference_config, inference_info)
+   
     cached_parameter_count['kv_cache'] = 2 * (inference_config['batchsize'] * (model_config['hidden_size'] * model_config['num_hidden_layers'] * (inference_config['input_seq_length']+inference_config['output_seq_length'])))
 
     operation_items = {key: "{:,}".format(int(generation_operation_count[key])) for key in generation_operation_count if key not in subtotal_operations}
@@ -199,10 +200,12 @@ with col4: # Generation
 
     header5("Summary: Generation")
     st.markdown(create_table(df_subtotal_operation_count))
-    st.write(f"Generation-only throughput (tokens/s): {inference_info['inference_generation_throughput']:.2f}")
-    st.write(f"(Client) Generation throughput (tokens/s): {inference_info['inference_client_generation_throughput']:.2f}")
+    #st.write(f"Generation-only throughput (tokens/s): {inference_info['inference_generation_throughput']:.2f}")
+    #st.write(f"(Client) Generation throughput (tokens/s): {inference_info['inference_client_generation_throughput']:.2f}")
     st.write(f"FLOPS latency: {inference_info['inference_generation_time']}")
     st.write(f"Memory latency: {inference_info['generation_memory_latency']}")
+    st.write(f"Generation-only throughput (tokens/s): {inference_info['generation_throughput']:.2f} ({inference_info['generation_bound_type']}-bound)")
+    st.write(f"(Client) Generation throughput (tokens/s): {inference_info['client_generation_throughput']:.2f}")
 
     if inference_config['KV_cache']:
         st.write(f"kv cache (Byte): {cached_parameter_count['kv_cache']:,}")

calc_util.py

@@ -296,4 +296,16 @@ def generation_activation_memory(model_config, inference_config):
         activation_memory['mlp'] + activation_memory['layernorm']
     )
 
-    return activation_memory
+    return activation_memory
+
+
+def calc_prefilling_throughput(model_config, inference_config, inference_info):
+    inference_info['prefilling_throughput'] = inference_config['input_seq_length']*inference_config['batchsize'] / max([inference_info['inference_prefilling_time'], inference_info['prefilling_memory_latency']])
+    inference_info['prefilling_bound_type'] = "memory" if inference_info['inference_prefilling_time'] < inference_info['prefilling_memory_latency'] else "arithmetic"
+
+def calc_generation_throughput(model_config, inference_config, inference_info):
+    inference_info['generation_throughput'] = inference_config['input_seq_length']*inference_config['batchsize'] / max([inference_info['inference_generation_time'], inference_info['generation_memory_latency']])
+    inference_info['generation_bound_type'] = "memory" if inference_info['inference_generation_time'] < inference_info['generation_memory_latency'] else "arithmetic"
+    
+    total_time = max([inference_info['inference_prefilling_time'], inference_info['prefilling_memory_latency']]) + max([inference_info['inference_generation_time'], inference_info['generation_memory_latency']])
+    inference_info['client_generation_throughput'] = inference_config['output_seq_length']*inference_config['batchsize'] / total_time