Just combine them. I think this should work but it's untested:
p <- ggplot(visual1, aes(ISSUE_DATE,COUNTED)) + geom_point() +
geom_smooth(fill="blue", colour="darkblue", size=1)
p <- p + geom_point(data=visual2, aes(ISSUE_DATE,COUNTED)) +
geom_smooth(data=visual2, fill="red", colour="red", size=1)
print(p)