simple way to prevent the whole app from getting effected by system font size is to updateConfiguration using a base activity.
//in base activity add this code.
public void adjustFontScale( Configuration configuration) {
configuration.fontScale = (float) 1.0;
DisplayMetrics metrics = getResources().getDisplayMetrics();
WindowManager wm = (WindowManager) getSystemService(WINDOW_SERVICE);
wm.getDefaultDisplay().getMetrics(metrics);
metrics.scaledDensity = configuration.fontScale * metrics.density;
getBaseContext().getResources().updateConfiguration(configuration, metrics);
}
@Override
protected void onCreate(@Nullable Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
adjustFontScale( getResources().getConfiguration());
}