Like sgibb said it was an if problem, it had nothing to do with | or ||.
Here is another way to solve your problem:
for (i in 1:nrow(trip)) {
if(trip$Ref.y[i]=='G' & trip$Variant.y[i]=='T'|trip$Ref.y[i]=='C' & trip$Variant.y[i]=='A') {
trip[i, 'mutType'] <- "G:C to T:A"
}
else if(trip$Ref.y[i]=='G' & trip$Variant.y[i]=='C'|trip$Ref.y[i]=='C' & trip$Variant.y[i]=='G') {
trip[i, 'mutType'] <- "G:C to C:G"
}
else if(trip$Ref.y[i]=='G' & trip$Variant.y[i]=='A'|trip$Ref.y[i]=='C' & trip$Variant.y[i]=='T') {
trip[i, 'mutType'] <- "G:C to A:T"
}
else if(trip$Ref.y[i]=='A' & trip$Variant.y[i]=='T'|trip$Ref.y[i]=='T' & trip$Variant.y[i]=='A') {
trip[i, 'mutType'] <- "A:T to T:A"
}
else if(trip$Ref.y[i]=='A' & trip$Variant.y[i]=='G'|trip$Ref.y[i]=='T' & trip$Variant.y[i]=='C') {
trip[i, 'mutType'] <- "A:T to G:C"
}
else if(trip$Ref.y[i]=='A' & trip$Variant.y[i]=='C'|trip$Ref.y[i]=='T' & trip$Variant.y[i]=='G') {
trip[i, 'mutType'] <- "A:T to C:G"
}
}