forked from ccf-ai-infra/GPUCodeForces
47 lines
1.7 KiB
Python
47 lines
1.7 KiB
Python
import torch
|
|
import torch.nn as nn
|
|
from torch.utils.cpp_extension import load_inline
|
|
|
|
cuda_source = """
|
|
#include <torch/extension.h>
|
|
#include <cuda_runtime.h>
|
|
|
|
__global__ void td_loss_kernel(const float* values, const float* rewards, const float* next_values, const float* dones, float* output, float gamma, int size) {
|
|
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (idx < size) {
|
|
float td_target = rewards[idx] + gamma * next_values[idx] * (1.0f - dones[idx]);
|
|
float td_error = values[idx] - td_target;
|
|
output[idx] = td_error * td_error;
|
|
}
|
|
}
|
|
|
|
torch::Tensor td_loss_cuda(torch::Tensor values, torch::Tensor rewards, torch::Tensor next_values, torch::Tensor dones, float gamma) {
|
|
auto size = values.numel();
|
|
auto output = torch::empty_like(values);
|
|
const int block_size = 256;
|
|
int num_blocks = (size + block_size - 1) / block_size;
|
|
td_loss_kernel<<<num_blocks, block_size>>>(values.data_ptr<float>(), rewards.data_ptr<float>(), next_values.data_ptr<float>(), dones.data_ptr<float>(), output.data_ptr<float>(), gamma, size);
|
|
return output.mean();
|
|
}
|
|
"""
|
|
|
|
cpp_source = """
|
|
torch::Tensor td_loss_cuda(torch::Tensor values, torch::Tensor rewards, torch::Tensor next_values, torch::Tensor dones, float gamma);
|
|
"""
|
|
|
|
td_module = load_inline(
|
|
name="td_loss",
|
|
cpp_sources=cpp_source,
|
|
cuda_sources=cuda_source,
|
|
functions=["td_loss_cuda"],
|
|
verbose=True
|
|
)
|
|
|
|
class ModelNew(torch.nn.Module):
|
|
def __init__(self, gamma):
|
|
super(ModelNew, self).__init__()
|
|
self.gamma = gamma
|
|
self.td_module = td_module
|
|
|
|
def forward(self, values, rewards, next_values, dones):
|
|
return self.td_module.td_loss_cuda(values, rewards, next_values, dones, self.gamma) |