If you use Material Components:
CustomDialog.kt
class CustomDialog: DialogFragment() {
override fun getTheme() = R.style.RoundedCornersDialog
}
styles.xml
<style name="RoundedCornersDialog" parent="Theme.MaterialComponents.Dialog">
<item name="dialogCornerRadius">dimen</item>
</style>