Try
byte[] incomingByteArray = receive...; // This is your Base64-encoded bute[]
byte[] decodedByteArray =Convert.FromBase64String (Encoding.ASCII.GetString (incomingByteArray));
// This work because all Base64-encoding is done with pure ASCII characters