diff --git a/doc/src/pair_buck.rst b/doc/src/pair_buck.rst index 8725a9e1ee..401cd58bd9 100644 --- a/doc/src/pair_buck.rst +++ b/doc/src/pair_buck.rst @@ -151,7 +151,7 @@ commands: * cutoff2 (distance units) The second coefficient, :math:`\rho`, must be greater than zero. -The coefficients A,:math:`\rho`, and C can be written as analytical expressions +The coefficients A, :math:`\rho`, and C can be written as analytical expressions of :math:`\epsilon` and :math:`\sigma`, in analogy to the Lennard-Jones potential :ref:`(Khrapak) `. diff --git a/doc/src/pair_colloid.rst b/doc/src/pair_colloid.rst index eb02695aec..e60a326642 100644 --- a/doc/src/pair_colloid.rst +++ b/doc/src/pair_colloid.rst @@ -77,7 +77,7 @@ The colloid-solvent interaction energy is given by \left[ 1 - \frac{\left(5 ~ a^6+45~a^4~r^2+63~a^2~r^4+15~r^6\right) \sigma^6} {15 \left(a-r\right)^6 \left( a+r \right)^6} \right], \quad r < r_c -where :math:A_{cs}` is the Hamaker constant, *a* is the radius of the colloidal +where :math:`A_{cs}` is the Hamaker constant, *a* is the radius of the colloidal particle, and :math:`r_c` is the cutoff. This formula is derived from the colloid-colloid interaction, letting one of the particle sizes go to zero. diff --git a/doc/src/pair_cosine_squared.rst b/doc/src/pair_cosine_squared.rst index a8cf206135..42e48a6641 100644 --- a/doc/src/pair_cosine_squared.rst +++ b/doc/src/pair_cosine_squared.rst @@ -54,7 +54,7 @@ between two point particles, where (:math:`\sigma, -\epsilon`) is the location of the (rightmost) minimum of the potential, as explained in the syntax section above. -This potential was first used in (Cooke)_#CKD for a coarse-grained lipid +This potential was first used in :ref:`(Cooke) ` for a coarse-grained lipid membrane model. It is generally very useful as a non-specific interaction potential because it is fully adjustable in depth and width while joining the minimum at (sigma, -epsilon) and zero at (cutoff, 0) @@ -63,7 +63,7 @@ energy calculations etc. This evidently requires *cutoff* to be larger than *sigma*\ . If the *wca* option is used then a Weeks-Chandler-Andersen potential -(Weeks)_#WCA is added to the above specified cosine-squared potential, +:ref:`(Weeks) ` is added to the above specified cosine-squared potential, specifically the following: .. math:: diff --git a/doc/src/pair_mdf.rst b/doc/src/pair_mdf.rst index 9cb9eaa6b2..8636802af3 100644 --- a/doc/src/pair_mdf.rst +++ b/doc/src/pair_mdf.rst @@ -40,7 +40,7 @@ Examples pair_coeff * * 1.0 1.0 pair_coeff 1 1 1.1 2.8 3.0 3.2 - pair_style buck 2.5 3.0 + pair_style buck/mdf 2.5 3.0 pair_coeff * * 100.0 1.5 200.0 pair_coeff * * 100.0 1.5 200.0 3.0 3.5 diff --git a/src/KOKKOS/angle_charmm_kokkos.cpp b/src/KOKKOS/angle_charmm_kokkos.cpp index 8d7a4d23d6..288e5a475b 100644 --- a/src/KOKKOS/angle_charmm_kokkos.cpp +++ b/src/KOKKOS/angle_charmm_kokkos.cpp @@ -446,7 +446,7 @@ void AngleCharmmKokkos::ev_tally(EV_FLOAT &ev, const int i, const in namespace LAMMPS_NS { template class AngleCharmmKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class AngleCharmmKokkos; #endif } diff --git a/src/KOKKOS/angle_class2_kokkos.cpp b/src/KOKKOS/angle_class2_kokkos.cpp index 50d7391ec5..ebb016b31b 100644 --- a/src/KOKKOS/angle_class2_kokkos.cpp +++ b/src/KOKKOS/angle_class2_kokkos.cpp @@ -597,7 +597,7 @@ void AngleClass2Kokkos::ev_tally(EV_FLOAT &ev, const int i, const in namespace LAMMPS_NS { template class AngleClass2Kokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class AngleClass2Kokkos; #endif } diff --git a/src/KOKKOS/angle_cosine_kokkos.cpp b/src/KOKKOS/angle_cosine_kokkos.cpp index afb955694d..960988b3b5 100644 --- a/src/KOKKOS/angle_cosine_kokkos.cpp +++ b/src/KOKKOS/angle_cosine_kokkos.cpp @@ -386,7 +386,7 @@ void AngleCosineKokkos::ev_tally(EV_FLOAT &ev, const int i, const in namespace LAMMPS_NS { template class AngleCosineKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class AngleCosineKokkos; #endif } diff --git a/src/KOKKOS/angle_harmonic_kokkos.cpp b/src/KOKKOS/angle_harmonic_kokkos.cpp index 4500e9b313..bc08ff4b65 100644 --- a/src/KOKKOS/angle_harmonic_kokkos.cpp +++ b/src/KOKKOS/angle_harmonic_kokkos.cpp @@ -404,7 +404,7 @@ void AngleHarmonicKokkos::ev_tally(EV_FLOAT &ev, const int i, const namespace LAMMPS_NS { template class AngleHarmonicKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class AngleHarmonicKokkos; #endif } diff --git a/src/KOKKOS/bond_class2_kokkos.cpp b/src/KOKKOS/bond_class2_kokkos.cpp index 514c80ddfb..88504608f6 100644 --- a/src/KOKKOS/bond_class2_kokkos.cpp +++ b/src/KOKKOS/bond_class2_kokkos.cpp @@ -365,7 +365,7 @@ void BondClass2Kokkos::ev_tally(EV_FLOAT &ev, const int &i, const in namespace LAMMPS_NS { template class BondClass2Kokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class BondClass2Kokkos; #endif } diff --git a/src/KOKKOS/bond_fene_kokkos.cpp b/src/KOKKOS/bond_fene_kokkos.cpp index 864873343d..e6aa27649f 100644 --- a/src/KOKKOS/bond_fene_kokkos.cpp +++ b/src/KOKKOS/bond_fene_kokkos.cpp @@ -400,7 +400,7 @@ void BondFENEKokkos::ev_tally(EV_FLOAT &ev, const int &i, const int namespace LAMMPS_NS { template class BondFENEKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class BondFENEKokkos; #endif } diff --git a/src/KOKKOS/bond_harmonic_kokkos.cpp b/src/KOKKOS/bond_harmonic_kokkos.cpp index 4e2a5aaa05..ee1b58c91f 100644 --- a/src/KOKKOS/bond_harmonic_kokkos.cpp +++ b/src/KOKKOS/bond_harmonic_kokkos.cpp @@ -339,7 +339,7 @@ void BondHarmonicKokkos::ev_tally(EV_FLOAT &ev, const int &i, const namespace LAMMPS_NS { template class BondHarmonicKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class BondHarmonicKokkos; #endif } diff --git a/src/KOKKOS/compute_coord_atom_kokkos.cpp b/src/KOKKOS/compute_coord_atom_kokkos.cpp index d91bf5a82a..90bd151b2d 100644 --- a/src/KOKKOS/compute_coord_atom_kokkos.cpp +++ b/src/KOKKOS/compute_coord_atom_kokkos.cpp @@ -243,7 +243,7 @@ void ComputeCoordAtomKokkos::operator()(TagComputeCoordAtom; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class ComputeCoordAtomKokkos; #endif } diff --git a/src/KOKKOS/compute_orientorder_atom_kokkos.cpp b/src/KOKKOS/compute_orientorder_atom_kokkos.cpp index fa613ad699..cda8f72250 100644 --- a/src/KOKKOS/compute_orientorder_atom_kokkos.cpp +++ b/src/KOKKOS/compute_orientorder_atom_kokkos.cpp @@ -737,7 +737,7 @@ void ComputeOrientOrderAtomKokkos::check_team_size_for(int inum, int namespace LAMMPS_NS { template class ComputeOrientOrderAtomKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class ComputeOrientOrderAtomKokkos; #endif } diff --git a/src/KOKKOS/compute_temp_kokkos.cpp b/src/KOKKOS/compute_temp_kokkos.cpp index f5547d0200..0e7fe25540 100644 --- a/src/KOKKOS/compute_temp_kokkos.cpp +++ b/src/KOKKOS/compute_temp_kokkos.cpp @@ -154,7 +154,7 @@ void ComputeTempKokkos::operator()(TagComputeTempVector, cons namespace LAMMPS_NS { template class ComputeTempKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class ComputeTempKokkos; #endif } diff --git a/src/KOKKOS/dihedral_charmm_kokkos.cpp b/src/KOKKOS/dihedral_charmm_kokkos.cpp index 64ab79c305..0d8256f306 100644 --- a/src/KOKKOS/dihedral_charmm_kokkos.cpp +++ b/src/KOKKOS/dihedral_charmm_kokkos.cpp @@ -789,7 +789,7 @@ void DihedralCharmmKokkos::ev_tally(EVM_FLOAT &evm, const int i, con namespace LAMMPS_NS { template class DihedralCharmmKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class DihedralCharmmKokkos; #endif } diff --git a/src/KOKKOS/dihedral_class2_kokkos.cpp b/src/KOKKOS/dihedral_class2_kokkos.cpp index adda9c56a9..371af44e9a 100644 --- a/src/KOKKOS/dihedral_class2_kokkos.cpp +++ b/src/KOKKOS/dihedral_class2_kokkos.cpp @@ -1131,7 +1131,7 @@ void DihedralClass2Kokkos::ev_tally(EV_FLOAT &ev, const int i1, cons namespace LAMMPS_NS { template class DihedralClass2Kokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class DihedralClass2Kokkos; #endif } diff --git a/src/KOKKOS/dihedral_harmonic_kokkos.cpp b/src/KOKKOS/dihedral_harmonic_kokkos.cpp index 15e459f0d5..408fa276e4 100644 --- a/src/KOKKOS/dihedral_harmonic_kokkos.cpp +++ b/src/KOKKOS/dihedral_harmonic_kokkos.cpp @@ -530,7 +530,7 @@ void DihedralHarmonicKokkos::ev_tally(EV_FLOAT &ev, const int i1, co namespace LAMMPS_NS { template class DihedralHarmonicKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class DihedralHarmonicKokkos; #endif } diff --git a/src/KOKKOS/dihedral_opls_kokkos.cpp b/src/KOKKOS/dihedral_opls_kokkos.cpp index cc3cb321c0..309d617f0a 100644 --- a/src/KOKKOS/dihedral_opls_kokkos.cpp +++ b/src/KOKKOS/dihedral_opls_kokkos.cpp @@ -535,7 +535,7 @@ void DihedralOPLSKokkos::ev_tally(EV_FLOAT &ev, const int i1, const namespace LAMMPS_NS { template class DihedralOPLSKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class DihedralOPLSKokkos; #endif } diff --git a/src/KOKKOS/fft3d_kokkos.cpp b/src/KOKKOS/fft3d_kokkos.cpp index 04dd343af6..e49d780040 100644 --- a/src/KOKKOS/fft3d_kokkos.cpp +++ b/src/KOKKOS/fft3d_kokkos.cpp @@ -905,7 +905,7 @@ void FFT3dKokkos::fft_3d_1d_only_kokkos(typename FFT_AT::t_FFT_DATA_ namespace LAMMPS_NS { template class FFT3dKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FFT3dKokkos; #endif } diff --git a/src/KOKKOS/fftdata_kokkos.h b/src/KOKKOS/fftdata_kokkos.h index e68e888805..51c444c2a3 100644 --- a/src/KOKKOS/fftdata_kokkos.h +++ b/src/KOKKOS/fftdata_kokkos.h @@ -52,7 +52,7 @@ typedef double FFT_SCALAR; // CUFFT or KISSFFT, thus undefine all other // FFTs here, since they may be valid in fft3d.cpp -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA # if defined(FFT_FFTW) # undef FFT_FFTW # endif @@ -164,7 +164,7 @@ typedef tdual_int_64::t_dev_um t_int_64_um; }; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template <> struct FFTArrayTypes { diff --git a/src/KOKKOS/fix_dpd_energy_kokkos.cpp b/src/KOKKOS/fix_dpd_energy_kokkos.cpp index 20579a6dc5..6549430131 100644 --- a/src/KOKKOS/fix_dpd_energy_kokkos.cpp +++ b/src/KOKKOS/fix_dpd_energy_kokkos.cpp @@ -85,7 +85,7 @@ void FixDPDenergyKokkos::final_integrate() namespace LAMMPS_NS { template class FixDPDenergyKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixDPDenergyKokkos; #endif } diff --git a/src/KOKKOS/fix_enforce2d_kokkos.cpp b/src/KOKKOS/fix_enforce2d_kokkos.cpp index fca3cedb13..cc8314a00f 100644 --- a/src/KOKKOS/fix_enforce2d_kokkos.cpp +++ b/src/KOKKOS/fix_enforce2d_kokkos.cpp @@ -163,7 +163,7 @@ void FixEnforce2DKokkos::post_force_item( int i ) const namespace LAMMPS_NS { template class FixEnforce2DKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixEnforce2DKokkos; #endif } diff --git a/src/KOKKOS/fix_eos_table_rx_kokkos.cpp b/src/KOKKOS/fix_eos_table_rx_kokkos.cpp index 89561c9a92..21ee54df28 100644 --- a/src/KOKKOS/fix_eos_table_rx_kokkos.cpp +++ b/src/KOKKOS/fix_eos_table_rx_kokkos.cpp @@ -560,7 +560,7 @@ void FixEOStableRXKokkos::create_kokkos_tables() namespace LAMMPS_NS { template class FixEOStableRXKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixEOStableRXKokkos; #endif } diff --git a/src/KOKKOS/fix_freeze_kokkos.cpp b/src/KOKKOS/fix_freeze_kokkos.cpp index fb0c3841e6..70f3d3236d 100644 --- a/src/KOKKOS/fix_freeze_kokkos.cpp +++ b/src/KOKKOS/fix_freeze_kokkos.cpp @@ -102,6 +102,7 @@ double FixFreezeKokkos::compute_vector(int n) /* ---------------------------------------------------------------------- */ template +KOKKOS_INLINE_FUNCTION void FixFreezeKokkos::operator()(const int i, OriginalForce &original) const { if (mask[i] & groupbit) { original.values[0] += f(i,0); @@ -118,7 +119,7 @@ void FixFreezeKokkos::operator()(const int i, OriginalForce &origina namespace LAMMPS_NS { template class FixFreezeKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixFreezeKokkos; #endif } diff --git a/src/KOKKOS/fix_gravity_kokkos.cpp b/src/KOKKOS/fix_gravity_kokkos.cpp index e74595e598..4f8b57365e 100644 --- a/src/KOKKOS/fix_gravity_kokkos.cpp +++ b/src/KOKKOS/fix_gravity_kokkos.cpp @@ -115,7 +115,7 @@ void FixGravityKokkos::operator()(TagFixGravityMass, const int i, do namespace LAMMPS_NS { template class FixGravityKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixGravityKokkos; #endif } diff --git a/src/KOKKOS/fix_langevin_kokkos.cpp b/src/KOKKOS/fix_langevin_kokkos.cpp index 244b20ea18..a8cc072ffc 100644 --- a/src/KOKKOS/fix_langevin_kokkos.cpp +++ b/src/KOKKOS/fix_langevin_kokkos.cpp @@ -909,7 +909,7 @@ void FixLangevinKokkos::cleanup_copy() namespace LAMMPS_NS { template class FixLangevinKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixLangevinKokkos; #endif } diff --git a/src/KOKKOS/fix_momentum_kokkos.cpp b/src/KOKKOS/fix_momentum_kokkos.cpp index 2d4911bfda..9c9d6cb1cd 100644 --- a/src/KOKKOS/fix_momentum_kokkos.cpp +++ b/src/KOKKOS/fix_momentum_kokkos.cpp @@ -199,7 +199,7 @@ void FixMomentumKokkos::end_of_step() namespace LAMMPS_NS { template class FixMomentumKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixMomentumKokkos; #endif } diff --git a/src/KOKKOS/fix_neigh_history_kokkos.cpp b/src/KOKKOS/fix_neigh_history_kokkos.cpp index 8c33f6f2c3..848940c4b7 100644 --- a/src/KOKKOS/fix_neigh_history_kokkos.cpp +++ b/src/KOKKOS/fix_neigh_history_kokkos.cpp @@ -343,7 +343,7 @@ int FixNeighHistoryKokkos::unpack_exchange(int nlocal, double *buf) namespace LAMMPS_NS { template class FixNeighHistoryKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixNeighHistoryKokkos; #endif } diff --git a/src/KOKKOS/fix_nh_kokkos.cpp b/src/KOKKOS/fix_nh_kokkos.cpp index 578f2f5c70..46993994b1 100644 --- a/src/KOKKOS/fix_nh_kokkos.cpp +++ b/src/KOKKOS/fix_nh_kokkos.cpp @@ -735,7 +735,7 @@ void FixNHKokkos::pre_exchange() namespace LAMMPS_NS { template class FixNHKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixNHKokkos; #endif } diff --git a/src/KOKKOS/fix_nph_kokkos.cpp b/src/KOKKOS/fix_nph_kokkos.cpp index 3830860fd7..d667e29221 100644 --- a/src/KOKKOS/fix_nph_kokkos.cpp +++ b/src/KOKKOS/fix_nph_kokkos.cpp @@ -71,7 +71,7 @@ FixNPHKokkos::FixNPHKokkos(LAMMPS *lmp, int narg, char **arg) : namespace LAMMPS_NS { template class FixNPHKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixNPHKokkos; #endif } diff --git a/src/KOKKOS/fix_npt_kokkos.cpp b/src/KOKKOS/fix_npt_kokkos.cpp index 5b751c9e4d..47b3da4efe 100644 --- a/src/KOKKOS/fix_npt_kokkos.cpp +++ b/src/KOKKOS/fix_npt_kokkos.cpp @@ -71,7 +71,7 @@ FixNPTKokkos::FixNPTKokkos(LAMMPS *lmp, int narg, char **arg) : namespace LAMMPS_NS { template class FixNPTKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixNPTKokkos; #endif } diff --git a/src/KOKKOS/fix_nve_kokkos.cpp b/src/KOKKOS/fix_nve_kokkos.cpp index f94b68dbce..3198557ee7 100644 --- a/src/KOKKOS/fix_nve_kokkos.cpp +++ b/src/KOKKOS/fix_nve_kokkos.cpp @@ -172,7 +172,7 @@ void FixNVEKokkos::cleanup_copy() namespace LAMMPS_NS { template class FixNVEKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixNVEKokkos; #endif } diff --git a/src/KOKKOS/fix_nve_sphere_kokkos.cpp b/src/KOKKOS/fix_nve_sphere_kokkos.cpp index 95945a858f..13d9fef69c 100644 --- a/src/KOKKOS/fix_nve_sphere_kokkos.cpp +++ b/src/KOKKOS/fix_nve_sphere_kokkos.cpp @@ -149,7 +149,7 @@ void FixNVESphereKokkos::final_integrate_item(const int i) const namespace LAMMPS_NS { template class FixNVESphereKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixNVESphereKokkos; #endif } diff --git a/src/KOKKOS/fix_nvt_kokkos.cpp b/src/KOKKOS/fix_nvt_kokkos.cpp index 66165dd7bc..013095abc0 100644 --- a/src/KOKKOS/fix_nvt_kokkos.cpp +++ b/src/KOKKOS/fix_nvt_kokkos.cpp @@ -52,7 +52,7 @@ FixNVTKokkos::FixNVTKokkos(LAMMPS *lmp, int narg, char **arg) : namespace LAMMPS_NS { template class FixNVTKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixNVTKokkos; #endif } diff --git a/src/KOKKOS/fix_qeq_reax_kokkos.cpp b/src/KOKKOS/fix_qeq_reax_kokkos.cpp index d0aedd5fc2..7e798c740b 100644 --- a/src/KOKKOS/fix_qeq_reax_kokkos.cpp +++ b/src/KOKKOS/fix_qeq_reax_kokkos.cpp @@ -478,6 +478,7 @@ void FixQEqReaxKokkos::compute_h_item(int ii, int &m_fill, const boo template template +KOKKOS_INLINE_FUNCTION void FixQEqReaxKokkos::compute_h_team( const typename Kokkos::TeamPolicy::member_type &team, int atoms_per_team, int vector_length) const { @@ -1524,7 +1525,7 @@ int FixQEqReaxKokkos::unpack_exchange(int nlocal, double *buf) namespace LAMMPS_NS { template class FixQEqReaxKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixQEqReaxKokkos; #endif } diff --git a/src/KOKKOS/fix_rx_kokkos.cpp b/src/KOKKOS/fix_rx_kokkos.cpp index 2ec3e3a77c..a760309218 100644 --- a/src/KOKKOS/fix_rx_kokkos.cpp +++ b/src/KOKKOS/fix_rx_kokkos.cpp @@ -220,6 +220,7 @@ void FixRxKokkos::rk4(const double t_stop, double *y, double *rwork, template template +KOKKOS_INLINE_FUNCTION void FixRxKokkos::k_rk4(const double t_stop, VectorType& y, VectorType& rwork, UserDataType& userData) const { VectorType k1( rwork ); @@ -279,6 +280,7 @@ void FixRxKokkos::k_rk4(const double t_stop, VectorType& y, VectorTy template template +KOKKOS_INLINE_FUNCTION void FixRxKokkos::k_rkf45_step (const int neq, const double h, VectorType& y, VectorType& y_out, VectorType& rwk, UserDataType& userData) const { const double c21=0.25; @@ -384,6 +386,7 @@ void FixRxKokkos::k_rkf45_step (const int neq, const double h, Vecto template template +KOKKOS_INLINE_FUNCTION int FixRxKokkos::k_rkf45_h0 (const int neq, const double t, const double t_stop, const double hmin, const double hmax, @@ -479,6 +482,7 @@ int FixRxKokkos::k_rkf45_h0 template template +KOKKOS_INLINE_FUNCTION void FixRxKokkos::k_rkf45(const int neq, const double t_stop, VectorType& y, VectorType& rwork, UserDataType& userData, CounterType& counter) const { // Rounding coefficient. @@ -1049,6 +1053,7 @@ int FixRxKokkos::rhs_sparse(double t, const double *y, double *dydt, template template +KOKKOS_INLINE_FUNCTION int FixRxKokkos::k_rhs(double t, const VectorType& y, VectorType& dydt, UserDataType& userData) const { // Use the sparse format instead. @@ -1062,6 +1067,7 @@ int FixRxKokkos::k_rhs(double t, const VectorType& y, VectorType& dy template template +KOKKOS_INLINE_FUNCTION int FixRxKokkos::k_rhs_dense(double t, const VectorType& y, VectorType& dydt, UserDataType& userData) const { #define rxnRateLaw (userData.rxnRateLaw) @@ -1103,6 +1109,7 @@ int FixRxKokkos::k_rhs_dense(double t, const VectorType& y, VectorTy template template +KOKKOS_INLINE_FUNCTION int FixRxKokkos::k_rhs_sparse(double t, const VectorType& y, VectorType& dydt, UserDataType& userData) const { #define kFor (userData.kFor) @@ -2275,7 +2282,7 @@ void FixRxKokkos::unpack_reverse_comm(int n, int *list, double *buf) namespace LAMMPS_NS { template class FixRxKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixRxKokkos; #endif } diff --git a/src/KOKKOS/fix_setforce_kokkos.cpp b/src/KOKKOS/fix_setforce_kokkos.cpp index 4763f04e70..da516f5ba2 100644 --- a/src/KOKKOS/fix_setforce_kokkos.cpp +++ b/src/KOKKOS/fix_setforce_kokkos.cpp @@ -184,7 +184,7 @@ void FixSetForceKokkos::operator()(TagFixSetForceNonConstant, const namespace LAMMPS_NS { template class FixSetForceKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixSetForceKokkos; #endif } diff --git a/src/KOKKOS/fix_shardlow_kokkos.cpp b/src/KOKKOS/fix_shardlow_kokkos.cpp index c6ad47501a..c44294b2ff 100644 --- a/src/KOKKOS/fix_shardlow_kokkos.cpp +++ b/src/KOKKOS/fix_shardlow_kokkos.cpp @@ -800,7 +800,7 @@ double FixShardlowKokkos::memory_usage() namespace LAMMPS_NS { template class FixShardlowKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixShardlowKokkos; #endif } diff --git a/src/KOKKOS/fix_wall_lj93_kokkos.cpp b/src/KOKKOS/fix_wall_lj93_kokkos.cpp index 61346144c8..4acf16c15b 100644 --- a/src/KOKKOS/fix_wall_lj93_kokkos.cpp +++ b/src/KOKKOS/fix_wall_lj93_kokkos.cpp @@ -98,7 +98,7 @@ void FixWallLJ93Kokkos::wall_particle_item(int i, value_type ewall) namespace LAMMPS_NS { template class FixWallLJ93Kokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixWallLJ93Kokkos; #endif } diff --git a/src/KOKKOS/fix_wall_reflect_kokkos.cpp b/src/KOKKOS/fix_wall_reflect_kokkos.cpp index ba104d19a9..a5bd6e6e6c 100644 --- a/src/KOKKOS/fix_wall_reflect_kokkos.cpp +++ b/src/KOKKOS/fix_wall_reflect_kokkos.cpp @@ -107,7 +107,7 @@ void FixWallReflectKokkos::operator()(TagFixWallReflectPostIntegrate namespace LAMMPS_NS { template class FixWallReflectKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class FixWallReflectKokkos; #endif } diff --git a/src/KOKKOS/gridcomm_kokkos.cpp b/src/KOKKOS/gridcomm_kokkos.cpp index bdf816b647..024428366e 100644 --- a/src/KOKKOS/gridcomm_kokkos.cpp +++ b/src/KOKKOS/gridcomm_kokkos.cpp @@ -652,7 +652,7 @@ double GridCommKokkos::memory_usage() namespace LAMMPS_NS { template class GridCommKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class GridCommKokkos; #endif } diff --git a/src/KOKKOS/improper_class2_kokkos.cpp b/src/KOKKOS/improper_class2_kokkos.cpp index fcc9592b19..320caecc83 100644 --- a/src/KOKKOS/improper_class2_kokkos.cpp +++ b/src/KOKKOS/improper_class2_kokkos.cpp @@ -1126,7 +1126,7 @@ void ImproperClass2Kokkos::ev_tally(EV_FLOAT &ev, const int i1, cons namespace LAMMPS_NS { template class ImproperClass2Kokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class ImproperClass2Kokkos; #endif } diff --git a/src/KOKKOS/improper_harmonic_kokkos.cpp b/src/KOKKOS/improper_harmonic_kokkos.cpp index 6f2969722a..124e56120b 100644 --- a/src/KOKKOS/improper_harmonic_kokkos.cpp +++ b/src/KOKKOS/improper_harmonic_kokkos.cpp @@ -477,7 +477,7 @@ void ImproperHarmonicKokkos::ev_tally(EV_FLOAT &ev, const int i1, co namespace LAMMPS_NS { template class ImproperHarmonicKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class ImproperHarmonicKokkos; #endif } diff --git a/src/KOKKOS/kissfft_kokkos.cpp b/src/KOKKOS/kissfft_kokkos.cpp index 5cfb3aa64c..ac8b1ef9cc 100644 --- a/src/KOKKOS/kissfft_kokkos.cpp +++ b/src/KOKKOS/kissfft_kokkos.cpp @@ -14,7 +14,7 @@ namespace LAMMPS_NS { template class KissFFTKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class KissFFTKokkos; #endif } diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp index f54df5cb76..b1c46698ec 100644 --- a/src/KOKKOS/kokkos.cpp +++ b/src/KOKKOS/kokkos.cpp @@ -100,8 +100,8 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) } else if (strcmp(arg[iarg],"g") == 0 || strcmp(arg[iarg],"gpus") == 0) { -#ifndef KOKKOS_ENABLE_CUDA - error->all(FLERR,"GPUs are requested but Kokkos has not been compiled for CUDA"); +#ifndef LMP_KOKKOS_GPU + error->all(FLERR,"GPUs are requested but Kokkos has not been compiled for CUDA or HIP"); #endif if (iarg+2 > narg) error->all(FLERR,"Invalid Kokkos command-line args"); ngpus = atoi(arg[iarg+1]); @@ -164,9 +164,9 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) if (logfile) fprintf(logfile," will use up to %d GPU(s) per node\n",ngpus); } -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU if (ngpus <= 0) - error->all(FLERR,"Kokkos has been compiled for CUDA but no GPUs are requested"); + error->all(FLERR,"Kokkos has been compiled for CUDA or HIP but no GPUs are requested"); #endif #ifndef KOKKOS_ENABLE_SERIAL diff --git a/src/KOKKOS/kokkos_type.h b/src/KOKKOS/kokkos_type.h index 3bd8ac3f1e..54488f790d 100644 --- a/src/KOKKOS/kokkos_type.h +++ b/src/KOKKOS/kokkos_type.h @@ -29,6 +29,10 @@ enum{FULL=1u,HALFTHREAD=2u,HALF=4u}; #define ISFINITE(x) std::isfinite(x) #endif +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#define LMP_KOKKOS_GPU +#endif + #define MAX_TYPES_STACKPARAMS 12 #define NeighClusterSize 8 @@ -186,6 +190,7 @@ template<> struct ExecutionSpaceFromDevice { static const LAMMPS_NS::ExecutionSpace space = LAMMPS_NS::Host; }; + #ifdef KOKKOS_ENABLE_CUDA template<> struct ExecutionSpaceFromDevice { @@ -193,6 +198,13 @@ struct ExecutionSpaceFromDevice { }; #endif +#if defined(KOKKOS_ENABLE_HIP) +template<> +struct ExecutionSpaceFromDevice { + static const LAMMPS_NS::ExecutionSpace space = LAMMPS_NS::Device; +}; +#endif + // Determine memory traits for force array // Do atomic trait when running HALFTHREAD neighbor list style @@ -221,6 +233,13 @@ struct AtomicDup { }; #endif +#if defined(KOKKOS_ENABLE_HIP) +template<> +struct AtomicDup { + enum {value = Kokkos::Experimental::ScatterAtomic}; +}; +#endif + #ifdef LMP_KOKKOS_USE_ATOMICS #ifdef KOKKOS_ENABLE_OPENMP @@ -753,7 +772,7 @@ typedef tdual_neighbors_2d::t_dev_const_randomread t_neighbors_2d_randomread; }; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template <> struct ArrayTypes { @@ -1087,7 +1106,7 @@ typedef SNAComplex SNAcomplex; #define ISFINITE(x) std::isfinite(x) #endif -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU #define LAMMPS_LAMBDA [=] __device__ #else #define LAMMPS_LAMBDA [=] diff --git a/src/KOKKOS/nbin_kokkos.cpp b/src/KOKKOS/nbin_kokkos.cpp index 5fc3a89732..9b41dc1bff 100644 --- a/src/KOKKOS/nbin_kokkos.cpp +++ b/src/KOKKOS/nbin_kokkos.cpp @@ -147,7 +147,7 @@ void NBinKokkos::binatomsItem(const int &i) const namespace LAMMPS_NS { template class NBinKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class NBinKokkos; #endif } diff --git a/src/KOKKOS/nbin_ssa_kokkos.cpp b/src/KOKKOS/nbin_ssa_kokkos.cpp index 0e46f7fc61..2d9411e707 100644 --- a/src/KOKKOS/nbin_ssa_kokkos.cpp +++ b/src/KOKKOS/nbin_ssa_kokkos.cpp @@ -298,7 +298,7 @@ void NBinSSAKokkos::sortBin( namespace LAMMPS_NS { template class NBinSSAKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class NBinSSAKokkos; #endif } diff --git a/src/KOKKOS/neigh_bond_kokkos.cpp b/src/KOKKOS/neigh_bond_kokkos.cpp index 34858ff1e1..b815a084fb 100644 --- a/src/KOKKOS/neigh_bond_kokkos.cpp +++ b/src/KOKKOS/neigh_bond_kokkos.cpp @@ -1305,7 +1305,7 @@ void NeighBondKokkos::update_domain_variables() namespace LAMMPS_NS { template class NeighBondKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class NeighBondKokkos; #endif } diff --git a/src/KOKKOS/neigh_list_kokkos.cpp b/src/KOKKOS/neigh_list_kokkos.cpp index afaf0dd1b8..ab0cec15aa 100644 --- a/src/KOKKOS/neigh_list_kokkos.cpp +++ b/src/KOKKOS/neigh_list_kokkos.cpp @@ -52,7 +52,7 @@ void NeighListKokkos::grow(int nmax) namespace LAMMPS_NS { template class NeighListKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class NeighListKokkos; #endif } diff --git a/src/KOKKOS/npair_copy_kokkos.cpp b/src/KOKKOS/npair_copy_kokkos.cpp index 016d68400f..98f4b29c1c 100644 --- a/src/KOKKOS/npair_copy_kokkos.cpp +++ b/src/KOKKOS/npair_copy_kokkos.cpp @@ -122,7 +122,7 @@ void NPairCopyKokkos::copy_to_cpu(NeighList *list) namespace LAMMPS_NS { template class NPairCopyKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class NPairCopyKokkos; #endif } diff --git a/src/KOKKOS/npair_halffull_kokkos.cpp b/src/KOKKOS/npair_halffull_kokkos.cpp index 754a5ca010..b109010935 100644 --- a/src/KOKKOS/npair_halffull_kokkos.cpp +++ b/src/KOKKOS/npair_halffull_kokkos.cpp @@ -123,7 +123,7 @@ void NPairHalffullKokkos::operator()(TagNPairHalffullCompute, namespace LAMMPS_NS { template class NPairHalffullKokkos; template class NPairHalffullKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class NPairHalffullKokkos; template class NPairHalffullKokkos; #endif diff --git a/src/KOKKOS/npair_kokkos.cpp b/src/KOKKOS/npair_kokkos.cpp index 0a2d05096c..66daccc981 100644 --- a/src/KOKKOS/npair_kokkos.cpp +++ b/src/KOKKOS/npair_kokkos.cpp @@ -1154,7 +1154,7 @@ template class NPairKokkos; template class NPairKokkos; template class NPairKokkos; template class NPairKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class NPairKokkos; template class NPairKokkos; template class NPairKokkos; diff --git a/src/KOKKOS/npair_skip_kokkos.cpp b/src/KOKKOS/npair_skip_kokkos.cpp index bb393c9b95..13080a9580 100644 --- a/src/KOKKOS/npair_skip_kokkos.cpp +++ b/src/KOKKOS/npair_skip_kokkos.cpp @@ -147,7 +147,7 @@ void NPairSkipKokkos::operator()(TagNPairSkipCountLocal, const int & namespace LAMMPS_NS { template class NPairSkipKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class NPairSkipKokkos; #endif } diff --git a/src/KOKKOS/npair_ssa_kokkos.cpp b/src/KOKKOS/npair_ssa_kokkos.cpp index b6d03e1445..55c7358042 100644 --- a/src/KOKKOS/npair_ssa_kokkos.cpp +++ b/src/KOKKOS/npair_ssa_kokkos.cpp @@ -525,6 +525,7 @@ fprintf(stdout, "Fina%03d %6d inum %6d gnum, total used %6d, allocated %6d\n" template +KOKKOS_INLINE_FUNCTION void NPairSSAKokkosExecute::build_locals_onePhase(const bool firstTry, int me, int workPhase) const { const typename ArrayTypes::t_int_1d_const_um stencil = d_stencil; @@ -661,6 +662,7 @@ fprintf(stdout, "Phas%03d phase %3d used %6d inums, workItems = %3d, skipped = % template +KOKKOS_INLINE_FUNCTION void NPairSSAKokkosExecute::build_ghosts_onePhase(int workPhase) const { const typename ArrayTypes::t_int_1d_const_um stencil = d_stencil; @@ -753,7 +755,7 @@ void NPairSSAKokkosExecute::build_ghosts_onePhase(int workPhase) con namespace LAMMPS_NS { template class NPairSSAKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class NPairSSAKokkos; #endif } diff --git a/src/KOKKOS/pair_buck_coul_cut_kokkos.cpp b/src/KOKKOS/pair_buck_coul_cut_kokkos.cpp index 46c5ee82cb..a6af4708d5 100644 --- a/src/KOKKOS/pair_buck_coul_cut_kokkos.cpp +++ b/src/KOKKOS/pair_buck_coul_cut_kokkos.cpp @@ -352,7 +352,7 @@ double PairBuckCoulCutKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairBuckCoulCutKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairBuckCoulCutKokkos; #endif } diff --git a/src/KOKKOS/pair_buck_coul_long_kokkos.cpp b/src/KOKKOS/pair_buck_coul_long_kokkos.cpp index 29aca38a77..cdf3b0a6c1 100644 --- a/src/KOKKOS/pair_buck_coul_long_kokkos.cpp +++ b/src/KOKKOS/pair_buck_coul_long_kokkos.cpp @@ -515,7 +515,7 @@ double PairBuckCoulLongKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairBuckCoulLongKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairBuckCoulLongKokkos; #endif } diff --git a/src/KOKKOS/pair_buck_kokkos.cpp b/src/KOKKOS/pair_buck_kokkos.cpp index 04ebd6608b..71f63a9f92 100644 --- a/src/KOKKOS/pair_buck_kokkos.cpp +++ b/src/KOKKOS/pair_buck_kokkos.cpp @@ -264,7 +264,7 @@ double PairBuckKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairBuckKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairBuckKokkos; #endif } diff --git a/src/KOKKOS/pair_coul_cut_kokkos.cpp b/src/KOKKOS/pair_coul_cut_kokkos.cpp index 2e3f76463d..2047286181 100644 --- a/src/KOKKOS/pair_coul_cut_kokkos.cpp +++ b/src/KOKKOS/pair_coul_cut_kokkos.cpp @@ -270,7 +270,7 @@ double PairCoulCutKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairCoulCutKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairCoulCutKokkos; #endif } diff --git a/src/KOKKOS/pair_coul_debye_kokkos.cpp b/src/KOKKOS/pair_coul_debye_kokkos.cpp index a544a5fb8d..cddfc63b4d 100644 --- a/src/KOKKOS/pair_coul_debye_kokkos.cpp +++ b/src/KOKKOS/pair_coul_debye_kokkos.cpp @@ -313,7 +313,7 @@ double PairCoulDebyeKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairCoulDebyeKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairCoulDebyeKokkos; #endif } diff --git a/src/KOKKOS/pair_coul_dsf_kokkos.cpp b/src/KOKKOS/pair_coul_dsf_kokkos.cpp index 62e205d8fd..11ca10c7be 100644 --- a/src/KOKKOS/pair_coul_dsf_kokkos.cpp +++ b/src/KOKKOS/pair_coul_dsf_kokkos.cpp @@ -414,7 +414,7 @@ int PairCoulDSFKokkos::sbmask(const int& j) const { namespace LAMMPS_NS { template class PairCoulDSFKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairCoulDSFKokkos; #endif } diff --git a/src/KOKKOS/pair_coul_long_kokkos.cpp b/src/KOKKOS/pair_coul_long_kokkos.cpp index c1f630a7a1..123763ea29 100644 --- a/src/KOKKOS/pair_coul_long_kokkos.cpp +++ b/src/KOKKOS/pair_coul_long_kokkos.cpp @@ -464,7 +464,7 @@ double PairCoulLongKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairCoulLongKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairCoulLongKokkos; #endif } diff --git a/src/KOKKOS/pair_coul_wolf_kokkos.cpp b/src/KOKKOS/pair_coul_wolf_kokkos.cpp index 907d088348..3bbb32a125 100644 --- a/src/KOKKOS/pair_coul_wolf_kokkos.cpp +++ b/src/KOKKOS/pair_coul_wolf_kokkos.cpp @@ -416,7 +416,7 @@ int PairCoulWolfKokkos::sbmask(const int& j) const { namespace LAMMPS_NS { template class PairCoulWolfKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairCoulWolfKokkos; #endif } diff --git a/src/KOKKOS/pair_dpd_fdt_energy_kokkos.cpp b/src/KOKKOS/pair_dpd_fdt_energy_kokkos.cpp index 61d2707888..f4dc0e867c 100644 --- a/src/KOKKOS/pair_dpd_fdt_energy_kokkos.cpp +++ b/src/KOKKOS/pair_dpd_fdt_energy_kokkos.cpp @@ -788,7 +788,7 @@ int PairDPDfdtEnergyKokkos::sbmask(const int& j) const { namespace LAMMPS_NS { template class PairDPDfdtEnergyKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairDPDfdtEnergyKokkos; #endif } diff --git a/src/KOKKOS/pair_eam_alloy_kokkos.cpp b/src/KOKKOS/pair_eam_alloy_kokkos.cpp index bea1b28544..96a07628e8 100644 --- a/src/KOKKOS/pair_eam_alloy_kokkos.cpp +++ b/src/KOKKOS/pair_eam_alloy_kokkos.cpp @@ -1223,7 +1223,7 @@ void PairEAMAlloyKokkos::file2array_alloy() namespace LAMMPS_NS { template class PairEAMAlloyKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairEAMAlloyKokkos; #endif } diff --git a/src/KOKKOS/pair_eam_fs_kokkos.cpp b/src/KOKKOS/pair_eam_fs_kokkos.cpp index 978883ab58..4f3302b455 100644 --- a/src/KOKKOS/pair_eam_fs_kokkos.cpp +++ b/src/KOKKOS/pair_eam_fs_kokkos.cpp @@ -1233,7 +1233,7 @@ void PairEAMFSKokkos::file2array_fs() namespace LAMMPS_NS { template class PairEAMFSKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairEAMFSKokkos; #endif } diff --git a/src/KOKKOS/pair_eam_kokkos.cpp b/src/KOKKOS/pair_eam_kokkos.cpp index 99a0756610..f0ac646b6b 100644 --- a/src/KOKKOS/pair_eam_kokkos.cpp +++ b/src/KOKKOS/pair_eam_kokkos.cpp @@ -896,7 +896,7 @@ void PairEAMKokkos::ev_tally(EV_FLOAT &ev, const int &i, const int & namespace LAMMPS_NS { template class PairEAMKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairEAMKokkos; #endif } diff --git a/src/KOKKOS/pair_exp6_rx_kokkos.cpp b/src/KOKKOS/pair_exp6_rx_kokkos.cpp index 8c98e32d5b..a41ef0e6d1 100644 --- a/src/KOKKOS/pair_exp6_rx_kokkos.cpp +++ b/src/KOKKOS/pair_exp6_rx_kokkos.cpp @@ -232,7 +232,7 @@ void PairExp6rxKokkos::compute(int eflag_in, int vflag_in) } else Kokkos::parallel_for(Kokkos::RangePolicy(0,np_total),*this); -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU Kokkos::parallel_for(Kokkos::RangePolicy(0,np_total),*this); #else int errorFlag = 0; @@ -277,7 +277,7 @@ void PairExp6rxKokkos::compute(int eflag_in, int vflag_in) EV_FLOAT ev; -#ifdef KOKKOS_ENABLE_CUDA // Use atomics +#ifdef LMP_KOKKOS_GPU // Use atomics if (neighflag == HALF) { if (newton_pair) { @@ -2654,7 +2654,7 @@ int PairExp6rxKokkos::sbmask(const int& j) const { namespace LAMMPS_NS { template class PairExp6rxKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairExp6rxKokkos; #endif } diff --git a/src/KOKKOS/pair_gran_hooke_history_kokkos.cpp b/src/KOKKOS/pair_gran_hooke_history_kokkos.cpp index bc5eb30845..07ef629e77 100644 --- a/src/KOKKOS/pair_gran_hooke_history_kokkos.cpp +++ b/src/KOKKOS/pair_gran_hooke_history_kokkos.cpp @@ -585,7 +585,7 @@ void PairGranHookeHistoryKokkos::ev_tally_xyz_atom(EV_FLOAT &ev, int namespace LAMMPS_NS { template class PairGranHookeHistoryKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairGranHookeHistoryKokkos; #endif } diff --git a/src/KOKKOS/pair_kokkos.h b/src/KOKKOS/pair_kokkos.h index 2934ddf621..06458948ef 100644 --- a/src/KOKKOS/pair_kokkos.h +++ b/src/KOKKOS/pair_kokkos.h @@ -720,7 +720,7 @@ int GetTeamSize(FunctorStyle& functor, int inum, int reduce_flag, int team_size, else team_size_max = Kokkos::TeamPolicy<>(inum,Kokkos::AUTO).team_size_max(functor,Kokkos::ParallelForTag()); -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU if(team_size*vector_length > team_size_max) team_size = team_size_max/vector_length; #else diff --git a/src/KOKKOS/pair_lj_charmm_coul_charmm_implicit_kokkos.cpp b/src/KOKKOS/pair_lj_charmm_coul_charmm_implicit_kokkos.cpp index 3996f49c29..c8c6e54a5f 100644 --- a/src/KOKKOS/pair_lj_charmm_coul_charmm_implicit_kokkos.cpp +++ b/src/KOKKOS/pair_lj_charmm_coul_charmm_implicit_kokkos.cpp @@ -520,7 +520,7 @@ double PairLJCharmmCoulCharmmImplicitKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJCharmmCoulCharmmImplicitKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJCharmmCoulCharmmImplicitKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_charmm_coul_charmm_kokkos.cpp b/src/KOKKOS/pair_lj_charmm_coul_charmm_kokkos.cpp index 44ba78fa7d..62adc84ab0 100644 --- a/src/KOKKOS/pair_lj_charmm_coul_charmm_kokkos.cpp +++ b/src/KOKKOS/pair_lj_charmm_coul_charmm_kokkos.cpp @@ -522,7 +522,7 @@ double PairLJCharmmCoulCharmmKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJCharmmCoulCharmmKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJCharmmCoulCharmmKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_charmm_coul_long_kokkos.cpp b/src/KOKKOS/pair_lj_charmm_coul_long_kokkos.cpp index 35724c7061..4c122773d7 100644 --- a/src/KOKKOS/pair_lj_charmm_coul_long_kokkos.cpp +++ b/src/KOKKOS/pair_lj_charmm_coul_long_kokkos.cpp @@ -529,7 +529,7 @@ double PairLJCharmmCoulLongKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJCharmmCoulLongKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJCharmmCoulLongKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_class2_coul_cut_kokkos.cpp b/src/KOKKOS/pair_lj_class2_coul_cut_kokkos.cpp index b8c72d7b88..4317029ab9 100644 --- a/src/KOKKOS/pair_lj_class2_coul_cut_kokkos.cpp +++ b/src/KOKKOS/pair_lj_class2_coul_cut_kokkos.cpp @@ -355,7 +355,7 @@ double PairLJClass2CoulCutKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJClass2CoulCutKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJClass2CoulCutKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_class2_coul_long_kokkos.cpp b/src/KOKKOS/pair_lj_class2_coul_long_kokkos.cpp index 1e01a275a4..51193bf692 100644 --- a/src/KOKKOS/pair_lj_class2_coul_long_kokkos.cpp +++ b/src/KOKKOS/pair_lj_class2_coul_long_kokkos.cpp @@ -510,7 +510,7 @@ double PairLJClass2CoulLongKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJClass2CoulLongKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJClass2CoulLongKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_class2_kokkos.cpp b/src/KOKKOS/pair_lj_class2_kokkos.cpp index 5bac4c8140..a0d2fba581 100644 --- a/src/KOKKOS/pair_lj_class2_kokkos.cpp +++ b/src/KOKKOS/pair_lj_class2_kokkos.cpp @@ -282,7 +282,7 @@ double PairLJClass2Kokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJClass2Kokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJClass2Kokkos; #endif } diff --git a/src/KOKKOS/pair_lj_cut_coul_cut_kokkos.cpp b/src/KOKKOS/pair_lj_cut_coul_cut_kokkos.cpp index e2b57e8df7..ebfb484836 100644 --- a/src/KOKKOS/pair_lj_cut_coul_cut_kokkos.cpp +++ b/src/KOKKOS/pair_lj_cut_coul_cut_kokkos.cpp @@ -347,7 +347,7 @@ double PairLJCutCoulCutKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJCutCoulCutKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJCutCoulCutKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_cut_coul_debye_kokkos.cpp b/src/KOKKOS/pair_lj_cut_coul_debye_kokkos.cpp index bc8369fd65..0e5b5999cd 100644 --- a/src/KOKKOS/pair_lj_cut_coul_debye_kokkos.cpp +++ b/src/KOKKOS/pair_lj_cut_coul_debye_kokkos.cpp @@ -376,7 +376,7 @@ double PairLJCutCoulDebyeKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJCutCoulDebyeKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJCutCoulDebyeKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_cut_coul_dsf_kokkos.cpp b/src/KOKKOS/pair_lj_cut_coul_dsf_kokkos.cpp index feb6a58136..8690859cd8 100644 --- a/src/KOKKOS/pair_lj_cut_coul_dsf_kokkos.cpp +++ b/src/KOKKOS/pair_lj_cut_coul_dsf_kokkos.cpp @@ -369,7 +369,7 @@ double PairLJCutCoulDSFKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJCutCoulDSFKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJCutCoulDSFKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_cut_coul_long_kokkos.cpp b/src/KOKKOS/pair_lj_cut_coul_long_kokkos.cpp index 595f018857..8cbfe03597 100644 --- a/src/KOKKOS/pair_lj_cut_coul_long_kokkos.cpp +++ b/src/KOKKOS/pair_lj_cut_coul_long_kokkos.cpp @@ -510,7 +510,7 @@ double PairLJCutCoulLongKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJCutCoulLongKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJCutCoulLongKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_cut_kokkos.cpp b/src/KOKKOS/pair_lj_cut_kokkos.cpp index 5eb0a5b91a..8294d62f68 100644 --- a/src/KOKKOS/pair_lj_cut_kokkos.cpp +++ b/src/KOKKOS/pair_lj_cut_kokkos.cpp @@ -278,7 +278,7 @@ double PairLJCutKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJCutKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJCutKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_expand_kokkos.cpp b/src/KOKKOS/pair_lj_expand_kokkos.cpp index 0409eda401..93385b7c70 100644 --- a/src/KOKKOS/pair_lj_expand_kokkos.cpp +++ b/src/KOKKOS/pair_lj_expand_kokkos.cpp @@ -287,7 +287,7 @@ double PairLJExpandKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJExpandKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJExpandKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_gromacs_coul_gromacs_kokkos.cpp b/src/KOKKOS/pair_lj_gromacs_coul_gromacs_kokkos.cpp index de475fbc99..fa9ca453a8 100644 --- a/src/KOKKOS/pair_lj_gromacs_coul_gromacs_kokkos.cpp +++ b/src/KOKKOS/pair_lj_gromacs_coul_gromacs_kokkos.cpp @@ -509,7 +509,7 @@ double PairLJGromacsCoulGromacsKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJGromacsCoulGromacsKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJGromacsCoulGromacsKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_gromacs_kokkos.cpp b/src/KOKKOS/pair_lj_gromacs_kokkos.cpp index fa2ab64ece..ed1e38a346 100644 --- a/src/KOKKOS/pair_lj_gromacs_kokkos.cpp +++ b/src/KOKKOS/pair_lj_gromacs_kokkos.cpp @@ -341,7 +341,7 @@ double PairLJGromacsKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJGromacsKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJGromacsKokkos; #endif } diff --git a/src/KOKKOS/pair_lj_sdk_kokkos.cpp b/src/KOKKOS/pair_lj_sdk_kokkos.cpp index 70d04385d0..8b228f2c01 100644 --- a/src/KOKKOS/pair_lj_sdk_kokkos.cpp +++ b/src/KOKKOS/pair_lj_sdk_kokkos.cpp @@ -317,7 +317,7 @@ double PairLJSDKKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairLJSDKKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairLJSDKKokkos; #endif } diff --git a/src/KOKKOS/pair_morse_kokkos.cpp b/src/KOKKOS/pair_morse_kokkos.cpp index 16d4d8a0a3..ba6e9ba97d 100644 --- a/src/KOKKOS/pair_morse_kokkos.cpp +++ b/src/KOKKOS/pair_morse_kokkos.cpp @@ -295,7 +295,7 @@ double PairMorseKokkos::init_one(int i, int j) namespace LAMMPS_NS { template class PairMorseKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairMorseKokkos; #endif } diff --git a/src/KOKKOS/pair_multi_lucy_rx_kokkos.cpp b/src/KOKKOS/pair_multi_lucy_rx_kokkos.cpp index edf8bb38ae..ca609f72b7 100644 --- a/src/KOKKOS/pair_multi_lucy_rx_kokkos.cpp +++ b/src/KOKKOS/pair_multi_lucy_rx_kokkos.cpp @@ -987,7 +987,7 @@ void PairMultiLucyRXKokkos::settings(int narg, char **arg) namespace LAMMPS_NS { template class PairMultiLucyRXKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairMultiLucyRXKokkos; #endif } diff --git a/src/KOKKOS/pair_reaxc_kokkos.cpp b/src/KOKKOS/pair_reaxc_kokkos.cpp index 2fdaddcf8d..201167f992 100644 --- a/src/KOKKOS/pair_reaxc_kokkos.cpp +++ b/src/KOKKOS/pair_reaxc_kokkos.cpp @@ -3958,7 +3958,7 @@ void PairReaxCKokkos::operator()(PairReaxFindBondSpecies, const int } template class PairReaxCKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairReaxCKokkos; #endif } diff --git a/src/KOKKOS/pair_snap_kokkos.cpp b/src/KOKKOS/pair_snap_kokkos.cpp index 76d6aa5462..baccd0476a 100644 --- a/src/KOKKOS/pair_snap_kokkos.cpp +++ b/src/KOKKOS/pair_snap_kokkos.cpp @@ -16,7 +16,7 @@ namespace LAMMPS_NS { template class PairSNAPKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairSNAPKokkos; #endif } diff --git a/src/KOKKOS/pair_snap_kokkos_impl.h b/src/KOKKOS/pair_snap_kokkos_impl.h index 35da0e8502..74bc26ef17 100644 --- a/src/KOKKOS/pair_snap_kokkos_impl.h +++ b/src/KOKKOS/pair_snap_kokkos_impl.h @@ -296,7 +296,7 @@ void PairSNAPKokkos::compute(int eflag_in, int vflag_in) } } else { // GPU -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU //PreUi { int vector_length = vector_length_default; diff --git a/src/KOKKOS/pair_sw_kokkos.cpp b/src/KOKKOS/pair_sw_kokkos.cpp index 4ca75ebd2e..4b5e9eafe2 100644 --- a/src/KOKKOS/pair_sw_kokkos.cpp +++ b/src/KOKKOS/pair_sw_kokkos.cpp @@ -967,7 +967,7 @@ void PairSWKokkos::ev_tally3_atom(EV_FLOAT &ev, const int &i, namespace LAMMPS_NS { template class PairSWKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairSWKokkos; #endif } diff --git a/src/KOKKOS/pair_table_kokkos.cpp b/src/KOKKOS/pair_table_kokkos.cpp index b2aaaf1566..721deb84f3 100644 --- a/src/KOKKOS/pair_table_kokkos.cpp +++ b/src/KOKKOS/pair_table_kokkos.cpp @@ -530,7 +530,7 @@ void PairTableKokkos::cleanup_copy() { namespace LAMMPS_NS { template class PairTableKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairTableKokkos; #endif diff --git a/src/KOKKOS/pair_table_rx_kokkos.cpp b/src/KOKKOS/pair_table_rx_kokkos.cpp index 2b066d7633..0efef97eb5 100644 --- a/src/KOKKOS/pair_table_rx_kokkos.cpp +++ b/src/KOKKOS/pair_table_rx_kokkos.cpp @@ -1297,7 +1297,7 @@ void PairTableRXKokkos::cleanup_copy() { namespace LAMMPS_NS { template class PairTableRXKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairTableRXKokkos; #endif diff --git a/src/KOKKOS/pair_tersoff_kokkos.cpp b/src/KOKKOS/pair_tersoff_kokkos.cpp index 134431aa44..910e8da4f3 100644 --- a/src/KOKKOS/pair_tersoff_kokkos.cpp +++ b/src/KOKKOS/pair_tersoff_kokkos.cpp @@ -1282,7 +1282,7 @@ int PairTersoffKokkos::sbmask(const int& j) const { namespace LAMMPS_NS { template class PairTersoffKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairTersoffKokkos; #endif } diff --git a/src/KOKKOS/pair_tersoff_mod_kokkos.cpp b/src/KOKKOS/pair_tersoff_mod_kokkos.cpp index ceb1ed1383..d51983bfc8 100644 --- a/src/KOKKOS/pair_tersoff_mod_kokkos.cpp +++ b/src/KOKKOS/pair_tersoff_mod_kokkos.cpp @@ -1285,7 +1285,7 @@ int PairTersoffMODKokkos::sbmask(const int& j) const { namespace LAMMPS_NS { template class PairTersoffMODKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairTersoffMODKokkos; #endif } diff --git a/src/KOKKOS/pair_tersoff_zbl_kokkos.cpp b/src/KOKKOS/pair_tersoff_zbl_kokkos.cpp index c7a08772c8..9c8bb6e1a5 100644 --- a/src/KOKKOS/pair_tersoff_zbl_kokkos.cpp +++ b/src/KOKKOS/pair_tersoff_zbl_kokkos.cpp @@ -1381,7 +1381,7 @@ int PairTersoffZBLKokkos::sbmask(const int& j) const { namespace LAMMPS_NS { template class PairTersoffZBLKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairTersoffZBLKokkos; #endif } diff --git a/src/KOKKOS/pair_vashishta_kokkos.cpp b/src/KOKKOS/pair_vashishta_kokkos.cpp index 1d46fe3cd2..47ffe64105 100644 --- a/src/KOKKOS/pair_vashishta_kokkos.cpp +++ b/src/KOKKOS/pair_vashishta_kokkos.cpp @@ -942,7 +942,7 @@ void PairVashishtaKokkos::ev_tally3_atom(EV_FLOAT &ev, const int &i, namespace LAMMPS_NS { template class PairVashishtaKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairVashishtaKokkos; #endif } diff --git a/src/KOKKOS/pair_yukawa_kokkos.cpp b/src/KOKKOS/pair_yukawa_kokkos.cpp index 5b641791d3..71236c807c 100644 --- a/src/KOKKOS/pair_yukawa_kokkos.cpp +++ b/src/KOKKOS/pair_yukawa_kokkos.cpp @@ -290,7 +290,7 @@ compute_evdwl(const F_FLOAT& rsq, const int& i, const int&j, namespace LAMMPS_NS { template class PairYukawaKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairYukawaKokkos; #endif } diff --git a/src/KOKKOS/pair_zbl_kokkos.cpp b/src/KOKKOS/pair_zbl_kokkos.cpp index 5e4ec398d9..8b583a75f7 100644 --- a/src/KOKKOS/pair_zbl_kokkos.cpp +++ b/src/KOKKOS/pair_zbl_kokkos.cpp @@ -423,7 +423,7 @@ void PairZBLKokkos::cleanup_copy() { namespace LAMMPS_NS { template class PairZBLKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PairZBLKokkos; #endif } diff --git a/src/KOKKOS/pppm_kokkos.cpp b/src/KOKKOS/pppm_kokkos.cpp index 8a87773816..743f2daec9 100644 --- a/src/KOKKOS/pppm_kokkos.cpp +++ b/src/KOKKOS/pppm_kokkos.cpp @@ -1627,7 +1627,7 @@ void PPPMKokkos::make_rho() nlocal = atomKK->nlocal; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU copymode = 1; Kokkos::parallel_for(Kokkos::RangePolicy(0,nlocal),*this); copymode = 0; @@ -3051,7 +3051,7 @@ double PPPMKokkos::memory_usage() namespace LAMMPS_NS { template class PPPMKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class PPPMKokkos; #endif } diff --git a/src/KOKKOS/pppm_kokkos.h b/src/KOKKOS/pppm_kokkos.h index 847a2f807f..56cd6f5140 100644 --- a/src/KOKKOS/pppm_kokkos.h +++ b/src/KOKKOS/pppm_kokkos.h @@ -31,7 +31,7 @@ KSpaceStyle(pppm/kk/host,PPPMKokkos) // fix up FFT defines for KOKKOS with CUDA -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA # if defined(FFT_FFTW) # undef FFT_FFTW # endif diff --git a/src/KOKKOS/rand_pool_wrap_kokkos.h b/src/KOKKOS/rand_pool_wrap_kokkos.h index c1461c5216..9124d8e9e7 100644 --- a/src/KOKKOS/rand_pool_wrap_kokkos.h +++ b/src/KOKKOS/rand_pool_wrap_kokkos.h @@ -50,7 +50,7 @@ class RandPoolWrap : protected Pointers { KOKKOS_INLINE_FUNCTION RandWrap get_state() const { -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU error->all(FLERR,"Cannot use Marsaglia RNG with GPUs"); #endif diff --git a/src/KOKKOS/region_block_kokkos.cpp b/src/KOKKOS/region_block_kokkos.cpp index b28230290e..da3122bd34 100644 --- a/src/KOKKOS/region_block_kokkos.cpp +++ b/src/KOKKOS/region_block_kokkos.cpp @@ -163,7 +163,7 @@ void RegBlockKokkos::rotate(double &x, double &y, double &z, double namespace LAMMPS_NS { template class RegBlockKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class RegBlockKokkos; #endif } diff --git a/src/KOKKOS/remap_kokkos.cpp b/src/KOKKOS/remap_kokkos.cpp index 604b438a89..2894009887 100644 --- a/src/KOKKOS/remap_kokkos.cpp +++ b/src/KOKKOS/remap_kokkos.cpp @@ -529,7 +529,7 @@ void RemapKokkos::remap_3d_destroy_plan_kokkos(struct remap_plan_3d_ namespace LAMMPS_NS { template class RemapKokkos; -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU template class RemapKokkos; #endif } diff --git a/src/KOKKOS/sna_kokkos_impl.h b/src/KOKKOS/sna_kokkos_impl.h index b017751c60..93d8347c5e 100644 --- a/src/KOKKOS/sna_kokkos_impl.h +++ b/src/KOKKOS/sna_kokkos_impl.h @@ -237,7 +237,7 @@ void SNAKokkos::grow_rij(int newnatom, int newnmax) element = t_sna_2i("sna:rcutij",natom,nmax); dedr = t_sna_3d("sna:dedr",natom,nmax,3); -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU if (std::is_same::value) { // dummy allocation ulisttot = t_sna_3c_ll("sna:ulisttot",1,1,1); @@ -269,7 +269,7 @@ void SNAKokkos::grow_rij(int newnatom, int newnmax) ylist_pack_im = t_sna_4d_ll("sna:ylist_pack_im",1,1,1,1); dulist = t_sna_4c3_ll("sna:dulist",idxu_max,natom,nmax); -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU } #endif } @@ -2029,7 +2029,7 @@ double SNAKokkos::memory_usage() bytes += jdimpq*jdimpq * sizeof(double); // pqarray bytes += idxcg_max * sizeof(double); // cglist -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU if (std::is_same::value) { int natom_pad = ((natom + 32 - 1) / 32) * 32; // for AoSoA layouts @@ -2055,7 +2055,7 @@ double SNAKokkos::memory_usage() bytes += natom * idxu_max * nelements * sizeof(double) * 2; // ylist bytes += natom * nmax * idxu_max * 3 * sizeof(double) * 2; // dulist -#ifdef KOKKOS_ENABLE_CUDA +#ifdef LMP_KOKKOS_GPU } #endif diff --git a/src/USER-MISC/pair_buck_mdf.cpp b/src/USER-MISC/pair_buck_mdf.cpp index 6339f7d135..1937aeedf7 100644 --- a/src/USER-MISC/pair_buck_mdf.cpp +++ b/src/USER-MISC/pair_buck_mdf.cpp @@ -285,6 +285,7 @@ void PairBuckMDF::write_restart(FILE *fp) fwrite(&rho[i][j],sizeof(double),1,fp); fwrite(&c[i][j],sizeof(double),1,fp); fwrite(&cut[i][j],sizeof(double),1,fp); + fwrite(&cut_inner[i][j],sizeof(double),1,fp); } } } @@ -311,11 +312,13 @@ void PairBuckMDF::read_restart(FILE *fp) utils::sfread(FLERR,&rho[i][j],sizeof(double),1,fp,NULL,error); utils::sfread(FLERR,&c[i][j],sizeof(double),1,fp,NULL,error); utils::sfread(FLERR,&cut[i][j],sizeof(double),1,fp,NULL,error); + utils::sfread(FLERR,&cut_inner[i][j],sizeof(double),1,fp,NULL,error); } MPI_Bcast(&a[i][j],1,MPI_DOUBLE,0,world); MPI_Bcast(&rho[i][j],1,MPI_DOUBLE,0,world); MPI_Bcast(&c[i][j],1,MPI_DOUBLE,0,world); MPI_Bcast(&cut[i][j],1,MPI_DOUBLE,0,world); + MPI_Bcast(&cut_inner[i][j],1,MPI_DOUBLE,0,world); } } } @@ -327,6 +330,7 @@ void PairBuckMDF::read_restart(FILE *fp) void PairBuckMDF::write_restart_settings(FILE *fp) { fwrite(&cut_global,sizeof(double),1,fp); + fwrite(&cut_inner_global,sizeof(double),1,fp); fwrite(&offset_flag,sizeof(int),1,fp); fwrite(&mix_flag,sizeof(int),1,fp); fwrite(&tail_flag,sizeof(int),1,fp); @@ -340,11 +344,13 @@ void PairBuckMDF::read_restart_settings(FILE *fp) { if (comm->me == 0) { utils::sfread(FLERR,&cut_global,sizeof(double),1,fp,NULL,error); + utils::sfread(FLERR,&cut_inner_global,sizeof(double),1,fp,NULL,error); utils::sfread(FLERR,&offset_flag,sizeof(int),1,fp,NULL,error); utils::sfread(FLERR,&mix_flag,sizeof(int),1,fp,NULL,error); utils::sfread(FLERR,&tail_flag,sizeof(int),1,fp,NULL,error); } MPI_Bcast(&cut_global,1,MPI_DOUBLE,0,world); + MPI_Bcast(&cut_inner_global,1,MPI_DOUBLE,0,world); MPI_Bcast(&offset_flag,1,MPI_INT,0,world); MPI_Bcast(&mix_flag,1,MPI_INT,0,world); MPI_Bcast(&tail_flag,1,MPI_INT,0,world); diff --git a/src/USER-MISC/pair_cosine_squared.cpp b/src/USER-MISC/pair_cosine_squared.cpp index 7f3f6259d4..ff0d763b2d 100644 --- a/src/USER-MISC/pair_cosine_squared.cpp +++ b/src/USER-MISC/pair_cosine_squared.cpp @@ -323,8 +323,8 @@ void PairCosineSquared::read_restart_settings(FILE *fp) void PairCosineSquared::write_data(FILE *fp) { for (int i = 1; i <= atom->ntypes; i++) - fprintf(fp, "%d %g %g %g %d\n", i, epsilon[i][i], sigma[i][i], - cut[i][i], wcaflag[i][i]); + fprintf(fp, "%d %g %g %g %s\n", i, epsilon[i][i], sigma[i][i], cut[i][i], + (wcaflag[i][i] ? "wca" : "")); } /* ---------------------------------------------------------------------- @@ -335,8 +335,8 @@ void PairCosineSquared::write_data_all(FILE *fp) { for (int i = 1; i <= atom->ntypes; i++) for (int j = i; j <= atom->ntypes; j++) - fprintf(fp, "%d %d %g %g %g %d\n", i, j, epsilon[i][j], sigma[i][j], - cut[i][j], wcaflag[i][j]); + fprintf(fp, "%d %d %g %g %g %s\n", i, j, epsilon[i][j], sigma[i][j], + cut[i][j], (wcaflag[i][j] ? "wca" : "")); } /* ---------------------------------------------------------------------- */ diff --git a/src/USER-OMP/ewald_omp.cpp b/src/USER-OMP/ewald_omp.cpp index fd776e46a6..f4473a6633 100644 --- a/src/USER-OMP/ewald_omp.cpp +++ b/src/USER-OMP/ewald_omp.cpp @@ -155,7 +155,7 @@ void EwaldOMP::compute(int eflag, int vflag) const double fac = qscale*q[i]; f[i][0] += fac*ek[i][0]; f[i][1] += fac*ek[i][1]; - f[i][2] += fac*ek[i][2]; + if (slabflag != 2) f[i][2] += fac*ek[i][2]; } // global energy @@ -222,7 +222,7 @@ void EwaldOMP::compute(int eflag, int vflag) virial[5] = v5 * qscale; } - if (slabflag) slabcorr(); + if (slabflag == 1) slabcorr(); } /* ---------------------------------------------------------------------- */ diff --git a/src/atom_vec.cpp b/src/atom_vec.cpp index 7b75499b83..261b068997 100644 --- a/src/atom_vec.cpp +++ b/src/atom_vec.cpp @@ -1096,8 +1096,9 @@ void AtomVec::unpack_border(int n, int first, double *buf) m = 0; last = first + n; + while (last > nmax) grow(0); + for (i = first; i < last; i++) { - if (i == nmax) grow(0); x[i][0] = buf[m++]; x[i][1] = buf[m++]; x[i][2] = buf[m++]; @@ -1165,8 +1166,9 @@ void AtomVec::unpack_border_vel(int n, int first, double *buf) m = 0; last = first + n; + while (last > nmax) grow(0); + for (i = first; i < last; i++) { - if (i == nmax) grow(0); x[i][0] = buf[m++]; x[i][1] = buf[m++]; x[i][2] = buf[m++]; diff --git a/src/comm_tiled.cpp b/src/comm_tiled.cpp index b7878ea86a..9f0169568d 100644 --- a/src/comm_tiled.cpp +++ b/src/comm_tiled.cpp @@ -11,6 +11,11 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + Contributing author (multi and triclinic support): + Adrian Diaz (University of Florida) +------------------------------------------------------------------------- */ + #include "comm_tiled.h" #include #include @@ -42,6 +47,11 @@ CommTiled::CommTiled(LAMMPS *lmp) : Comm(lmp) style = 1; layout = Comm::LAYOUT_UNIFORM; pbc_flag = NULL; + buf_send = NULL; + buf_recv = NULL; + overlap = NULL; + rcbinfo = NULL; + cutghostmulti = NULL; init_buffers(); } @@ -69,6 +79,7 @@ CommTiled::~CommTiled() memory->destroy(overlap); deallocate_swap(maxswap); memory->sfree(rcbinfo); + memory->destroy(cutghostmulti); } /* ---------------------------------------------------------------------- @@ -84,11 +95,12 @@ void CommTiled::init_buffers() maxoverlap = 0; overlap = NULL; + rcbinfo = NULL; + cutghostmulti = NULL; + sendbox_multi = NULL; maxswap = 6; allocate_swap(maxswap); - - rcbinfo = NULL; } /* ---------------------------------------------------------------------- */ @@ -97,16 +109,18 @@ void CommTiled::init() { Comm::init(); - nswap = 2 * domain->dimension; + // cannot set nswap in init_buffers() b/c + // dimension command can be after comm_style command + + nswap = 2*domain->dimension; + + memory->destroy(cutghostmulti); + if (mode == Comm::MULTI) + memory->create(cutghostmulti,atom->ntypes+1,3,"comm:cutghostmulti"); int bufextra_old = bufextra; init_exchange(); if (bufextra > bufextra_old) grow_send(maxsend+bufextra,2); - - // temporary restrictions - - if (mode == Comm::MULTI) - error->all(FLERR,"Cannot yet use comm_style tiled with multi-mode comm"); } /* ---------------------------------------------------------------------- @@ -122,6 +136,7 @@ void CommTiled::setup() dimension = domain->dimension; int *periodicity = domain->periodicity; + int ntypes = atom->ntypes; if (triclinic == 0) { prd = domain->prd; @@ -158,6 +173,18 @@ void CommTiled::setup() // set cutoff for comm forward and comm reverse // check that cutoff < any periodic box length + if (mode == Comm::MULTI) { + double *cuttype = neighbor->cuttype; + double cut; + for (i = 1; i <= ntypes; i++) { + cut = 0.0; + if (cutusermulti) cut = cutusermulti[i]; + cutghostmulti[i][0] = MAX(cut,cuttype[i]); + cutghostmulti[i][1] = MAX(cut,cuttype[i]); + cutghostmulti[i][2] = MAX(cut,cuttype[i]); + } + } + double cut = get_comm_cutoff(); if ((cut == 0.0) && (me == 0)) error->warning(FLERR,"Communication cutoff is 0.0. No ghost atoms " @@ -173,6 +200,13 @@ void CommTiled::setup() cutghost[1] = cut * length1; length2 = h_inv[2]; cutghost[2] = cut * length2; + if (mode == Comm::MULTI){ + for (i = 1; i <= ntypes; i++) { + cutghostmulti[i][0] *= length0; + cutghostmulti[i][1] *= length1; + cutghostmulti[i][2] *= length2; + } + } } if ((periodicity[0] && cutghost[0] > prd[0]) || @@ -308,62 +342,140 @@ void CommTiled::setup() // = obox in other 2 dims // if sbox touches other proc's sub-box boundaries in lower dims, // extend sbox in those lower dims to include ghost atoms + // single mode and multi mode - double oboxlo[3],oboxhi[3],sbox[6]; + double oboxlo[3],oboxhi[3],sbox[6],sbox_multi[6]; - for (i = 0; i < noverlap; i++) { - pbc_flag[iswap][i] = 0; - pbc[iswap][i][0] = pbc[iswap][i][1] = pbc[iswap][i][2] = - pbc[iswap][i][3] = pbc[iswap][i][4] = pbc[iswap][i][5] = 0; + if (mode == Comm::SINGLE) { + for (i = 0; i < noverlap; i++) { + pbc_flag[iswap][i] = 0; + pbc[iswap][i][0] = pbc[iswap][i][1] = pbc[iswap][i][2] = + pbc[iswap][i][3] = pbc[iswap][i][4] = pbc[iswap][i][5] = 0; - (this->*box_other)(idim,idir,overlap[i],oboxlo,oboxhi); + (this->*box_other)(idim,idir,overlap[i],oboxlo,oboxhi); - if (i < noverlap1) { - sbox[0] = MAX(oboxlo[0],lo1[0]); - sbox[1] = MAX(oboxlo[1],lo1[1]); - sbox[2] = MAX(oboxlo[2],lo1[2]); - sbox[3] = MIN(oboxhi[0],hi1[0]); - sbox[4] = MIN(oboxhi[1],hi1[1]); - sbox[5] = MIN(oboxhi[2],hi1[2]); - - } else { - pbc_flag[iswap][i] = 1; - if (idir == 0) pbc[iswap][i][idim] = 1; - else pbc[iswap][i][idim] = -1; - if (triclinic) { - if (idim == 1) pbc[iswap][i][5] = pbc[iswap][i][idim]; - if (idim == 2) pbc[iswap][i][4] = pbc[iswap][i][3] = pbc[iswap][i][idim]; + if (i < noverlap1) { + sbox[0] = MAX(oboxlo[0],lo1[0]); + sbox[1] = MAX(oboxlo[1],lo1[1]); + sbox[2] = MAX(oboxlo[2],lo1[2]); + sbox[3] = MIN(oboxhi[0],hi1[0]); + sbox[4] = MIN(oboxhi[1],hi1[1]); + sbox[5] = MIN(oboxhi[2],hi1[2]); + } else { + pbc_flag[iswap][i] = 1; + if (idir == 0) pbc[iswap][i][idim] = 1; + else pbc[iswap][i][idim] = -1; + if (triclinic) { + if (idim == 1) pbc[iswap][i][5] = pbc[iswap][i][idim]; + if (idim == 2) pbc[iswap][i][4] = pbc[iswap][i][3] = pbc[iswap][i][idim]; + } + sbox[0] = MAX(oboxlo[0],lo2[0]); + sbox[1] = MAX(oboxlo[1],lo2[1]); + sbox[2] = MAX(oboxlo[2],lo2[2]); + sbox[3] = MIN(oboxhi[0],hi2[0]); + sbox[4] = MIN(oboxhi[1],hi2[1]); + sbox[5] = MIN(oboxhi[2],hi2[2]); } - sbox[0] = MAX(oboxlo[0],lo2[0]); - sbox[1] = MAX(oboxlo[1],lo2[1]); - sbox[2] = MAX(oboxlo[2],lo2[2]); - sbox[3] = MIN(oboxhi[0],hi2[0]); - sbox[4] = MIN(oboxhi[1],hi2[1]); - sbox[5] = MIN(oboxhi[2],hi2[2]); - } + if (idir == 0) { + sbox[idim] = sublo[idim]; + if (i < noverlap1) + sbox[3+idim] = MIN(sbox[3+idim]+cutghost[idim],subhi[idim]); + else + sbox[3+idim] = MIN(sbox[3+idim]-prd[idim]+cutghost[idim],subhi[idim]); + } else { + if (i < noverlap1) sbox[idim] = MAX(sbox[idim]-cutghost[idim],sublo[idim]); + else sbox[idim] = MAX(sbox[idim]+prd[idim]-cutghost[idim],sublo[idim]); + sbox[3+idim] = subhi[idim]; + } - if (idir == 0) { - sbox[idim] = sublo[idim]; - if (i < noverlap1) - sbox[3+idim] = MIN(sbox[3+idim]+cutghost[idim],subhi[idim]); - else sbox[3+idim] = MIN(sbox[3+idim]-prd[idim]+cutghost[idim],subhi[idim]); - } else { - if (i < noverlap1) sbox[idim] = MAX(sbox[idim]-cutghost[idim],sublo[idim]); - else sbox[idim] = MAX(sbox[idim]+prd[idim]-cutghost[idim],sublo[idim]); - sbox[3+idim] = subhi[idim]; - } + if (idim >= 1) { + if (sbox[0] == oboxlo[0]) sbox[0] -= cutghost[0]; + if (sbox[3] == oboxhi[0]) sbox[3] += cutghost[0]; + } + if (idim == 2) { + if (sbox[1] == oboxlo[1]) sbox[1] -= cutghost[1]; + if (sbox[4] == oboxhi[1]) sbox[4] += cutghost[1]; + } - if (idim >= 1) { - if (sbox[0] == oboxlo[0]) sbox[0] -= cutghost[0]; - if (sbox[3] == oboxhi[0]) sbox[3] += cutghost[0]; - } - if (idim == 2) { - if (sbox[1] == oboxlo[1]) sbox[1] -= cutghost[1]; - if (sbox[4] == oboxhi[1]) sbox[4] += cutghost[1]; + memcpy(sendbox[iswap][i],sbox,6*sizeof(double)); } + } - memcpy(sendbox[iswap][i],sbox,6*sizeof(double)); + if (mode == Comm::MULTI) { + for (i = 0; i < noverlap; i++) { + pbc_flag[iswap][i] = 0; + pbc[iswap][i][0] = pbc[iswap][i][1] = pbc[iswap][i][2] = + pbc[iswap][i][3] = pbc[iswap][i][4] = pbc[iswap][i][5] = 0; + + (this->*box_other)(idim,idir,overlap[i],oboxlo,oboxhi); + + if (i < noverlap1) { + sbox[0] = MAX(oboxlo[0],lo1[0]); + sbox[1] = MAX(oboxlo[1],lo1[1]); + sbox[2] = MAX(oboxlo[2],lo1[2]); + sbox[3] = MIN(oboxhi[0],hi1[0]); + sbox[4] = MIN(oboxhi[1],hi1[1]); + sbox[5] = MIN(oboxhi[2],hi1[2]); + } else { + pbc_flag[iswap][i] = 1; + if (idir == 0) pbc[iswap][i][idim] = 1; + else pbc[iswap][i][idim] = -1; + if (triclinic) { + if (idim == 1) pbc[iswap][i][5] = pbc[iswap][i][idim]; + if (idim == 2) pbc[iswap][i][4] = pbc[iswap][i][3] = pbc[iswap][i][idim]; + } + sbox[0] = MAX(oboxlo[0],lo2[0]); + sbox[1] = MAX(oboxlo[1],lo2[1]); + sbox[2] = MAX(oboxlo[2],lo2[2]); + sbox[3] = MIN(oboxhi[0],hi2[0]); + sbox[4] = MIN(oboxhi[1],hi2[1]); + sbox[5] = MIN(oboxhi[2],hi2[2]); + } + + for (int itype = 1; itype <= atom->ntypes; itype++) { + sbox_multi[0] = sbox[0]; + sbox_multi[1] = sbox[1]; + sbox_multi[2] = sbox[2]; + sbox_multi[3] = sbox[3]; + sbox_multi[4] = sbox[4]; + sbox_multi[5] = sbox[5]; + if (idir == 0) { + sbox_multi[idim] = sublo[idim]; + if (i < noverlap1) + sbox_multi[3+idim] = + MIN(sbox_multi[3+idim]+cutghostmulti[itype][idim],subhi[idim]); + else + sbox_multi[3+idim] = + MIN(sbox_multi[3+idim]-prd[idim]+cutghostmulti[itype][idim], + subhi[idim]); + } else { + if (i < noverlap1) + sbox_multi[idim] = + MAX(sbox_multi[idim]-cutghostmulti[itype][idim],sublo[idim]); + else + sbox_multi[idim] = + MAX(sbox_multi[idim]+prd[idim]-cutghostmulti[itype][idim], + sublo[idim]); + sbox_multi[3+idim] = subhi[idim]; + } + + if (idim >= 1) { + if (sbox_multi[0] == oboxlo[0]) + sbox_multi[0] -= cutghostmulti[itype][idim]; + if (sbox_multi[3] == oboxhi[0]) + sbox_multi[3] += cutghostmulti[itype][idim]; + } + if (idim == 2) { + if (sbox_multi[1] == oboxlo[1]) + sbox_multi[1] -= cutghostmulti[itype][idim]; + if (sbox_multi[4] == oboxhi[1]) + sbox_multi[4] += cutghostmulti[itype][idim]; + } + + memcpy(sendbox_multi[iswap][i][itype],sbox_multi,6*sizeof(double)); + } + } } iswap++; @@ -445,15 +557,15 @@ void CommTiled::setup() } } - // reallocate MPI Requests and Statuses as needed + // reallocate MPI Requests as needed int nmax = 0; for (i = 0; i < nswap; i++) nmax = MAX(nmax,nprocmax[i]); for (i = 0; i < dimension; i++) nmax = MAX(nmax,nexchprocmax[i]); - if (nmax > maxreqstat) { - maxreqstat = nmax; + if (nmax > maxrequest) { + maxrequest = nmax; delete [] requests; - requests = new MPI_Request[maxreqstat]; + requests = new MPI_Request[maxrequest]; } } @@ -815,7 +927,7 @@ void CommTiled::borders() // for x-dim swaps, check owned atoms // for yz-dim swaps, check owned and ghost atoms // store sent atom indices in sendlist for use in future timesteps - // NOTE: assume SINGLE mode, add logic for MULTI mode later + // single mode and multi mode x = atom->x; if (iswap % 2 == 0) nlast = atom->nlocal + atom->nghost; @@ -823,45 +935,100 @@ void CommTiled::borders() ncountall = 0; for (m = 0; m < nsendproc[iswap]; m++) { - bbox = sendbox[iswap][m]; - xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; - xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; - ncount = 0; + if (mode == Comm::SINGLE) { + bbox = sendbox[iswap][m]; + xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; + xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; - if (!bordergroup) { - for (i = 0; i < nlast; i++) { - if (x[i][0] >= xlo && x[i][0] < xhi && - x[i][1] >= ylo && x[i][1] < yhi && - x[i][2] >= zlo && x[i][2] < zhi) { - if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); - sendlist[iswap][m][ncount++] = i; + ncount = 0; + + if (!bordergroup) { + for (i = 0; i < nlast; i++) { + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + sendlist[iswap][m][ncount++] = i; + } + } + } else { + ngroup = atom->nfirst; + for (i = 0; i < ngroup; i++) { + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + sendlist[iswap][m][ncount++] = i; + } + } + for (i = atom->nlocal; i < nlast; i++) { + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + sendlist[iswap][m][ncount++] = i; + } } } + + sendnum[iswap][m] = ncount; + smaxone = MAX(smaxone,ncount); + ncountall += ncount; + } else { - ngroup = atom->nfirst; - for (i = 0; i < ngroup; i++) { - if (x[i][0] >= xlo && x[i][0] < xhi && - x[i][1] >= ylo && x[i][1] < yhi && - x[i][2] >= zlo && x[i][2] < zhi) { - if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); - sendlist[iswap][m][ncount++] = i; - } - } - for (i = atom->nlocal; i < nlast; i++) { - if (x[i][0] >= xlo && x[i][0] < xhi && - x[i][1] >= ylo && x[i][1] < yhi && - x[i][2] >= zlo && x[i][2] < zhi) { - if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); - sendlist[iswap][m][ncount++] = i; - } - } - } - sendnum[iswap][m] = ncount; - smaxone = MAX(smaxone,ncount); - ncountall += ncount; + int* type=atom->type; + int itype; + ncount = 0; + + if (!bordergroup) { + for (i = 0; i < nlast; i++) { + itype=type[i]; + bbox = sendbox_multi[iswap][m][itype]; + xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; + xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + sendlist[iswap][m][ncount++] = i; + } + } + } else { + ngroup = atom->nfirst; + for (i = 0; i < ngroup; i++) { + itype=type[i]; + bbox = sendbox_multi[iswap][m][itype]; + xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; + xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + sendlist[iswap][m][ncount++] = i; + } + } + for (i = atom->nlocal; i < nlast; i++) { + itype=type[i]; + bbox = sendbox_multi[iswap][m][itype]; + xlo = bbox[0]; ylo = bbox[1]; zlo = bbox[2]; + xhi = bbox[3]; yhi = bbox[4]; zhi = bbox[5]; + if (x[i][0] >= xlo && x[i][0] < xhi && + x[i][1] >= ylo && x[i][1] < yhi && + x[i][2] >= zlo && x[i][2] < zhi) { + if (ncount == maxsendlist[iswap][m]) grow_list(iswap,m,ncount); + sendlist[iswap][m][ncount++] = i; + } + } + } + + sendnum[iswap][m] = ncount; + smaxone = MAX(smaxone,ncount); + ncountall += ncount; + } } + smaxall = MAX(smaxall,ncountall); // send sendnum counts to procs who recv from me except self @@ -914,9 +1081,8 @@ void CommTiled::borders() if (rmaxall*size_border > maxrecv) grow_recv(rmaxall*size_border); // swap atoms with other procs using pack_border(), unpack_border() - // use Waitall() instead of Waitany() because calls to unpack_border() - // must increment per-atom arrays in ascending order - // For the same reason, sendself unpacks must occur after recvother unpacks + // can use Waitany() because calls to unpack_border() + // increment per-atom arrays as much as needed if (ghost_velocity) { if (recvother[iswap]) { @@ -932,13 +1098,6 @@ void CommTiled::borders() MPI_Send(buf_send,n,MPI_DOUBLE,sendproc[iswap][m],0,world); } } - if (recvother[iswap]) { - MPI_Waitall(nrecv,requests,MPI_STATUS_IGNORE); - for (m = 0; m < nrecv; m++) - avec->unpack_border_vel(recvnum[iswap][m],firstrecv[iswap][m], - &buf_recv[size_border* - forward_recv_offset[iswap][m]]); - } if (sendself[iswap]) { avec->pack_border_vel(sendnum[iswap][nsend],sendlist[iswap][nsend], buf_send,pbc_flag[iswap][nsend], @@ -946,6 +1105,14 @@ void CommTiled::borders() avec->unpack_border_vel(recvnum[iswap][nrecv],firstrecv[iswap][nrecv], buf_send); } + if (recvother[iswap]) { + for (i = 0; i < nrecv; i++) { + MPI_Waitany(nrecv,requests,&m,MPI_STATUS_IGNORE); + avec->unpack_border_vel(recvnum[iswap][m],firstrecv[iswap][m], + &buf_recv[size_border* + forward_recv_offset[iswap][m]]); + } + } } else { if (recvother[iswap]) { @@ -961,19 +1128,20 @@ void CommTiled::borders() MPI_Send(buf_send,n,MPI_DOUBLE,sendproc[iswap][m],0,world); } } - if (recvother[iswap]) { - MPI_Waitall(nrecv,requests,MPI_STATUS_IGNORE); - for (m = 0; m < nrecv; m++) - avec->unpack_border(recvnum[iswap][m],firstrecv[iswap][m], - &buf_recv[size_border* - forward_recv_offset[iswap][m]]); - } if (sendself[iswap]) { avec->pack_border(sendnum[iswap][nsend],sendlist[iswap][nsend], buf_send,pbc_flag[iswap][nsend],pbc[iswap][nsend]); avec->unpack_border(recvnum[iswap][nrecv],firstrecv[iswap][nrecv], buf_send); } + if (recvother[iswap]) { + for (i = 0; i < nrecv; i++) { + MPI_Waitany(nrecv,requests,&m,MPI_STATUS_IGNORE); + avec->unpack_border(recvnum[iswap][m],firstrecv[iswap][m], + &buf_recv[size_border* + forward_recv_offset[iswap][m]]); + } + } } // increment ghost atoms @@ -1476,11 +1644,9 @@ int CommTiled::exchange_variable(int n, double * /*inbuf*/, double *& /*outbuf*/ void CommTiled::box_drop_brick(int idim, double *lo, double *hi, int &indexme) { - // NOTE: this is not triclinic compatible - // NOTE: these error messages are internal sanity checks - // should not occur, can be removed at some point + int dir; + int index = -1; - int index=-1,dir; if (hi[idim] == sublo[idim]) { index = myloc[idim] - 1; dir = -1; @@ -1913,6 +2079,7 @@ void CommTiled::allocate_swap(int n) pbc_flag = new int*[n]; pbc = new int**[n]; sendbox = new double**[n]; + sendbox_multi = new double***[n]; maxsendlist = new int*[n]; sendlist = new int**[n]; @@ -1926,11 +2093,12 @@ void CommTiled::allocate_swap(int n) pbc_flag[i] = NULL; pbc[i] = NULL; sendbox[i] = NULL; + sendbox_multi[i] = NULL; maxsendlist[i] = NULL; sendlist[i] = NULL; } - maxreqstat = 0; + maxrequest = 0; requests = NULL; for (int i = 0; i < n; i++) { @@ -1974,6 +2142,8 @@ void CommTiled::grow_swap_send(int i, int n, int nold) memory->create(pbc[i],n,6,"comm:pbc_flag"); memory->destroy(sendbox[i]); memory->create(sendbox[i],n,6,"comm:sendbox"); + memory->destroy(sendbox_multi[i]); + memory->create(sendbox_multi[i],n,atom->ntypes+1,6,"comm:sendbox_multi"); delete [] maxsendlist[i]; maxsendlist[i] = new int[n]; @@ -2032,6 +2202,8 @@ void CommTiled::deallocate_swap(int n) delete [] pbc_flag[i]; memory->destroy(pbc[i]); memory->destroy(sendbox[i]); + memory->destroy(sendbox_multi[i]); + delete [] maxsendlist[i]; for (int j = 0; j < nprocmax[i]; j++) memory->destroy(sendlist[i][j]); @@ -2052,6 +2224,7 @@ void CommTiled::deallocate_swap(int n) delete [] pbc_flag; delete [] pbc; delete [] sendbox; + delete [] sendbox_multi; delete [] maxsendlist; delete [] sendlist; diff --git a/src/comm_tiled.h b/src/comm_tiled.h index 6a0e2b82e3..aa8867b2c3 100644 --- a/src/comm_tiled.h +++ b/src/comm_tiled.h @@ -55,7 +55,7 @@ class CommTiled : public Comm { private: int nswap; // # of swaps to perform = 2*dim int maxswap; // largest nswap can be = 6 - + // forward/reverse comm info, proc lists include self int *nsendproc,*nrecvproc; // # of procs to send/recv to/from per swap @@ -70,7 +70,6 @@ class CommTiled : public Comm { int **size_reverse_recv; // # of values to recv in each reverse swap/proc int **forward_recv_offset; // forward comm offsets in buf_recv per swap/proc int **reverse_recv_offset; // reverse comm offsets in buf_recv per swap/proc - int ***sendlist; // list of atoms to send per swap/proc int **maxsendlist; // max size of send list per swap/proc int **pbc_flag; // general flag for sending atoms thru PBC @@ -78,6 +77,10 @@ class CommTiled : public Comm { double ***sendbox; // bounding box of atoms to send per swap/proc + double **cutghostmulti; // cutghost on a per-type basis + double ****sendbox_multi; // bounding box of atoms to send + // per swap/proc for multi comm + // exchange comm info, proc lists do not include self int *nexchproc; // # of procs to send/recv to/from in each dim @@ -92,7 +95,7 @@ class CommTiled : public Comm { int smaxall,rmaxall; // max size in atoms of any borders send/recv // for comm to all procs in one swap - int maxreqstat; // max size of Request and Status vectors + int maxrequest; // max size of Request vector MPI_Request *requests; struct RCBinfo { @@ -147,6 +150,7 @@ class CommTiled : public Comm { void grow_swap_send(int, int, int); // grow swap arrays for send and recv void grow_swap_recv(int, int); void deallocate_swap(int); // deallocate swap arrays + }; } diff --git a/src/fmt/LICENSE b/src/fmt/LICENSE new file mode 100644 index 0000000000..403c99037f --- /dev/null +++ b/src/fmt/LICENSE @@ -0,0 +1,33 @@ +Applies to the following files: +LAMMPS: Original from fmtlib/fmt: +src/fmt/*.h include/fmt/*.h +src/fmtlib_os.cpp src/os.cc +src/fmtlib_format.cpp src/format.cc + +Copyright (c) 2012 - present, Victor Zverovich + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Optional exception to the license --- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into a machine-executable object form of such +source code, you may redistribute such embedded portions in such object form +without including the above copyright and permission notices. diff --git a/src/fmt/chrono.h b/src/fmt/chrono.h index 421d464ad8..e70b8053a6 100644 --- a/src/fmt/chrono.h +++ b/src/fmt/chrono.h @@ -48,7 +48,7 @@ FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) { // From fits in To without any problem. } else { // From does not always fit in To, resort to a dynamic check. - if (from < T::min() || from > T::max()) { + if (from < (T::min)() || from > (T::max)()) { // outside range. ec = 1; return {}; @@ -74,7 +74,7 @@ FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) { if (F::is_signed && !T::is_signed) { // From may be negative, not allowed! - if (fmt::internal::is_negative(from)) { + if (fmt::detail::is_negative(from)) { ec = 1; return {}; } @@ -84,7 +84,7 @@ FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) { // yes, From always fits in To. } else { // from may not fit in To, we have to do a dynamic check - if (from > static_cast(T::max())) { + if (from > static_cast((T::max)())) { ec = 1; return {}; } @@ -97,7 +97,7 @@ FMT_CONSTEXPR To lossless_integral_conversion(const From from, int& ec) { // yes, From always fits in To. } else { // from may not fit in To, we have to do a dynamic check - if (from > static_cast(T::max())) { + if (from > static_cast((T::max)())) { // outside range. ec = 1; return {}; @@ -141,7 +141,7 @@ FMT_CONSTEXPR To safe_float_conversion(const From from, int& ec) { // catch the only happy case if (std::isfinite(from)) { - if (from >= T::lowest() && from <= T::max()) { + if (from >= T::lowest() && from <= (T::max)()) { return static_cast(from); } // not within range. @@ -195,12 +195,13 @@ To safe_duration_cast(std::chrono::duration from, } // multiply with Factor::num without overflow or underflow if (Factor::num != 1) { - const auto max1 = internal::max_value() / Factor::num; + const auto max1 = detail::max_value() / Factor::num; if (count > max1) { ec = 1; return {}; } - const auto min1 = std::numeric_limits::min() / Factor::num; + const auto min1 = + (std::numeric_limits::min)() / Factor::num; if (count < min1) { ec = 1; return {}; @@ -269,7 +270,7 @@ To safe_duration_cast(std::chrono::duration from, // multiply with Factor::num without overflow or underflow if (Factor::num != 1) { - constexpr auto max1 = internal::max_value() / + constexpr auto max1 = detail::max_value() / static_cast(Factor::num); if (count > max1) { ec = 1; @@ -306,12 +307,12 @@ To safe_duration_cast(std::chrono::duration from, // Usage: f FMT_NOMACRO() #define FMT_NOMACRO -namespace internal { +namespace detail { inline null<> localtime_r FMT_NOMACRO(...) { return null<>(); } inline null<> localtime_s(...) { return null<>(); } inline null<> gmtime_r(...) { return null<>(); } inline null<> gmtime_s(...) { return null<>(); } -} // namespace internal +} // namespace detail // Thread-safe replacement for std::localtime inline std::tm localtime(std::time_t time) { @@ -322,22 +323,22 @@ inline std::tm localtime(std::time_t time) { dispatcher(std::time_t t) : time_(t) {} bool run() { - using namespace fmt::internal; + using namespace fmt::detail; return handle(localtime_r(&time_, &tm_)); } bool handle(std::tm* tm) { return tm != nullptr; } - bool handle(internal::null<>) { - using namespace fmt::internal; + bool handle(detail::null<>) { + using namespace fmt::detail; return fallback(localtime_s(&tm_, &time_)); } bool fallback(int res) { return res == 0; } #if !FMT_MSC_VER - bool fallback(internal::null<>) { - using namespace fmt::internal; + bool fallback(detail::null<>) { + using namespace fmt::detail; std::tm* tm = std::localtime(&time_); if (tm) tm_ = *tm; return tm != nullptr; @@ -359,21 +360,21 @@ inline std::tm gmtime(std::time_t time) { dispatcher(std::time_t t) : time_(t) {} bool run() { - using namespace fmt::internal; + using namespace fmt::detail; return handle(gmtime_r(&time_, &tm_)); } bool handle(std::tm* tm) { return tm != nullptr; } - bool handle(internal::null<>) { - using namespace fmt::internal; + bool handle(detail::null<>) { + using namespace fmt::detail; return fallback(gmtime_s(&tm_, &time_)); } bool fallback(int res) { return res == 0; } #if !FMT_MSC_VER - bool fallback(internal::null<>) { + bool fallback(detail::null<>) { std::tm* tm = std::gmtime(&time_); if (tm) tm_ = *tm; return tm != nullptr; @@ -386,17 +387,17 @@ inline std::tm gmtime(std::time_t time) { return gt.tm_; } -namespace internal { -inline std::size_t strftime(char* str, std::size_t count, const char* format, - const std::tm* time) { +namespace detail { +inline size_t strftime(char* str, size_t count, const char* format, + const std::tm* time) { return std::strftime(str, count, format, time); } -inline std::size_t strftime(wchar_t* str, std::size_t count, - const wchar_t* format, const std::tm* time) { +inline size_t strftime(wchar_t* str, size_t count, const wchar_t* format, + const std::tm* time) { return std::wcsftime(str, count, format, time); } -} // namespace internal +} // namespace detail template struct formatter { template @@ -405,7 +406,7 @@ template struct formatter { if (it != ctx.end() && *it == ':') ++it; auto end = it; while (end != ctx.end() && *end != '}') ++end; - tm_format.reserve(internal::to_unsigned(end - it + 1)); + tm_format.reserve(detail::to_unsigned(end - it + 1)); tm_format.append(it, end); tm_format.push_back('\0'); return end; @@ -414,11 +415,10 @@ template struct formatter { template auto format(const std::tm& tm, FormatContext& ctx) -> decltype(ctx.out()) { basic_memory_buffer buf; - std::size_t start = buf.size(); + size_t start = buf.size(); for (;;) { - std::size_t size = buf.capacity() - start; - std::size_t count = - internal::strftime(&buf[start], size, &tm_format[0], &tm); + size_t size = buf.capacity() - start; + size_t count = detail::strftime(&buf[start], size, &tm_format[0], &tm); if (count != 0) { buf.resize(start + count); break; @@ -430,7 +430,7 @@ template struct formatter { // https://github.com/fmtlib/fmt/issues/367 break; } - const std::size_t MIN_GROWTH = 10; + const size_t MIN_GROWTH = 10; buf.reserve(buf.capacity() + (size > MIN_GROWTH ? size : MIN_GROWTH)); } return std::copy(buf.begin(), buf.end(), ctx.out()); @@ -439,7 +439,7 @@ template struct formatter { basic_memory_buffer tm_format; }; -namespace internal { +namespace detail { template FMT_CONSTEXPR const char* get_units() { return nullptr; } @@ -768,19 +768,25 @@ OutputIt format_duration_value(OutputIt out, Rep val, int precision) { return format_to(out, std::is_floating_point::value ? fp_f : format, val); } +template +OutputIt copy_unit(string_view unit, OutputIt out, Char) { + return std::copy(unit.begin(), unit.end(), out); +} + +template +OutputIt copy_unit(string_view unit, OutputIt out, wchar_t) { + // This works when wchar_t is UTF-32 because units only contain characters + // that have the same representation in UTF-16 and UTF-32. + utf8_to_utf16 u(unit); + return std::copy(u.c_str(), u.c_str() + u.size(), out); +} template OutputIt format_duration_unit(OutputIt out) { - if (const char* unit = get_units()) { - string_view s(unit); - if (const_check(std::is_same())) { - utf8_to_utf16 u(s); - return std::copy(u.c_str(), u.c_str() + u.size(), out); - } - return std::copy(s.begin(), s.end(), out); - } + if (const char* unit = get_units()) + return copy_unit(string_view(unit), out, Char()); const Char num_f[] = {'[', '{', '}', ']', 's', 0}; - if (Period::den == 1) return format_to(out, num_f, Period::num); + if (const_check(Period::den == 1)) return format_to(out, num_f, Period::num); const Char num_def_f[] = {'[', '{', '}', '/', '{', '}', ']', 's', 0}; return format_to(out, num_def_f, Period::num, Period::den); } @@ -874,9 +880,9 @@ struct chrono_formatter { if (isnan(value)) return write_nan(); uint32_or_64_or_128_t n = to_unsigned(to_nonnegative_int(value, max_value())); - int num_digits = internal::count_digits(n); + int num_digits = detail::count_digits(n); if (width > num_digits) out = std::fill_n(out, width - num_digits, '0'); - out = format_decimal(out, n, num_digits); + out = format_decimal(out, n, num_digits).end; } void write_nan() { std::copy_n("nan", 3, out); } @@ -1004,14 +1010,14 @@ struct chrono_formatter { out = format_duration_unit(out); } }; -} // namespace internal +} // namespace detail template struct formatter, Char> { private: basic_format_specs specs; int precision; - using arg_ref_type = internal::arg_ref; + using arg_ref_type = detail::arg_ref; arg_ref_type width_ref; arg_ref_type precision_ref; mutable basic_string_view format_str; @@ -1032,7 +1038,7 @@ struct formatter, Char> { return arg_ref_type(arg_id); } - FMT_CONSTEXPR arg_ref_type make_arg_ref(internal::auto_id) { + FMT_CONSTEXPR arg_ref_type make_arg_ref(detail::auto_id) { return arg_ref_type(context.next_arg_id()); } @@ -1062,17 +1068,17 @@ struct formatter, Char> { auto begin = ctx.begin(), end = ctx.end(); if (begin == end || *begin == '}') return {begin, begin}; spec_handler handler{*this, ctx, format_str}; - begin = internal::parse_align(begin, end, handler); + begin = detail::parse_align(begin, end, handler); if (begin == end) return {begin, begin}; - begin = internal::parse_width(begin, end, handler); + begin = detail::parse_width(begin, end, handler); if (begin == end) return {begin, begin}; if (*begin == '.') { if (std::is_floating_point::value) - begin = internal::parse_precision(begin, end, handler); + begin = detail::parse_precision(begin, end, handler); else handler.on_error("precision not allowed for this argument type"); } - end = parse_chrono_format(begin, end, internal::chrono_format_checker()); + end = parse_chrono_format(begin, end, detail::chrono_format_checker()); return {begin, end}; } @@ -1083,7 +1089,7 @@ struct formatter, Char> { -> decltype(ctx.begin()) { auto range = do_parse(ctx); format_str = basic_string_view( - &*range.begin, internal::to_unsigned(range.end - range.begin)); + &*range.begin, detail::to_unsigned(range.end - range.begin)); return range.end; } @@ -1094,23 +1100,21 @@ struct formatter, Char> { // is not specified. basic_memory_buffer buf; auto out = std::back_inserter(buf); - using range = internal::output_range; - internal::basic_writer w(range(ctx.out())); - internal::handle_dynamic_spec(specs.width, - width_ref, ctx); - internal::handle_dynamic_spec( - precision, precision_ref, ctx); + detail::handle_dynamic_spec(specs.width, width_ref, + ctx); + detail::handle_dynamic_spec(precision, + precision_ref, ctx); if (begin == end || *begin == '}') { - out = internal::format_duration_value(out, d.count(), precision); - internal::format_duration_unit(out); + out = detail::format_duration_value(out, d.count(), precision); + detail::format_duration_unit(out); } else { - internal::chrono_formatter f( + detail::chrono_formatter f( ctx, out, d); f.precision = precision; parse_chrono_format(begin, end, f); } - w.write(buf.data(), buf.size(), specs); - return w.out(); + return detail::write( + ctx.out(), basic_string_view(buf.data(), buf.size()), specs); } }; diff --git a/src/fmt/color.h b/src/fmt/color.h index 96d9ab6b43..b65f892afc 100644 --- a/src/fmt/color.h +++ b/src/fmt/color.h @@ -198,7 +198,7 @@ struct rgb { uint8_t b; }; -namespace internal { +namespace detail { // color is a struct of either a rgb color or a terminal color. struct color_type { @@ -221,7 +221,7 @@ struct color_type { uint32_t rgb_color; } value; }; -} // namespace internal +} // namespace detail // Experimental text formatting support. class text_style { @@ -298,11 +298,11 @@ class text_style { FMT_CONSTEXPR bool has_emphasis() const FMT_NOEXCEPT { return static_cast(ems) != 0; } - FMT_CONSTEXPR internal::color_type get_foreground() const FMT_NOEXCEPT { + FMT_CONSTEXPR detail::color_type get_foreground() const FMT_NOEXCEPT { FMT_ASSERT(has_foreground(), "no foreground specified for this style"); return foreground_color; } - FMT_CONSTEXPR internal::color_type get_background() const FMT_NOEXCEPT { + FMT_CONSTEXPR detail::color_type get_background() const FMT_NOEXCEPT { FMT_ASSERT(has_background(), "no background specified for this style"); return background_color; } @@ -313,7 +313,7 @@ class text_style { private: FMT_CONSTEXPR text_style(bool is_foreground, - internal::color_type text_color) FMT_NOEXCEPT + detail::color_type text_color) FMT_NOEXCEPT : set_foreground_color(), set_background_color(), ems() { @@ -326,23 +326,23 @@ class text_style { } } - friend FMT_CONSTEXPR_DECL text_style fg(internal::color_type foreground) + friend FMT_CONSTEXPR_DECL text_style fg(detail::color_type foreground) FMT_NOEXCEPT; - friend FMT_CONSTEXPR_DECL text_style bg(internal::color_type background) + friend FMT_CONSTEXPR_DECL text_style bg(detail::color_type background) FMT_NOEXCEPT; - internal::color_type foreground_color; - internal::color_type background_color; + detail::color_type foreground_color; + detail::color_type background_color; bool set_foreground_color; bool set_background_color; emphasis ems; }; -FMT_CONSTEXPR text_style fg(internal::color_type foreground) FMT_NOEXCEPT { +FMT_CONSTEXPR text_style fg(detail::color_type foreground) FMT_NOEXCEPT { return text_style(/*is_foreground=*/true, foreground); } -FMT_CONSTEXPR text_style bg(internal::color_type background) FMT_NOEXCEPT { +FMT_CONSTEXPR text_style bg(detail::color_type background) FMT_NOEXCEPT { return text_style(/*is_foreground=*/false, background); } @@ -350,21 +350,21 @@ FMT_CONSTEXPR text_style operator|(emphasis lhs, emphasis rhs) FMT_NOEXCEPT { return text_style(lhs) | rhs; } -namespace internal { +namespace detail { template struct ansi_color_escape { - FMT_CONSTEXPR ansi_color_escape(internal::color_type text_color, + FMT_CONSTEXPR ansi_color_escape(detail::color_type text_color, const char* esc) FMT_NOEXCEPT { // If we have a terminal color, we need to output another escape code // sequence. if (!text_color.is_rgb) { - bool is_background = esc == internal::data::background_color; + bool is_background = esc == detail::data::background_color; uint32_t value = text_color.value.term_color; // Background ASCII codes are the same as the foreground ones but with // 10 more. if (is_background) value += 10u; - std::size_t index = 0; + size_t index = 0; buffer[index++] = static_cast('\x1b'); buffer[index++] = static_cast('['); @@ -398,7 +398,7 @@ template struct ansi_color_escape { if (em_bits & static_cast(emphasis::strikethrough)) em_codes[3] = 9; - std::size_t index = 0; + size_t index = 0; for (int i = 0; i < 4; ++i) { if (!em_codes[i]) continue; buffer[index++] = static_cast('\x1b'); @@ -429,14 +429,14 @@ template struct ansi_color_escape { template FMT_CONSTEXPR ansi_color_escape make_foreground_color( - internal::color_type foreground) FMT_NOEXCEPT { - return ansi_color_escape(foreground, internal::data::foreground_color); + detail::color_type foreground) FMT_NOEXCEPT { + return ansi_color_escape(foreground, detail::data::foreground_color); } template FMT_CONSTEXPR ansi_color_escape make_background_color( - internal::color_type background) FMT_NOEXCEPT { - return ansi_color_escape(background, internal::data::background_color); + detail::color_type background) FMT_NOEXCEPT { + return ansi_color_escape(background, detail::data::background_color); } template @@ -455,11 +455,11 @@ inline void fputs(const wchar_t* chars, FILE* stream) FMT_NOEXCEPT { } template inline void reset_color(FILE* stream) FMT_NOEXCEPT { - fputs(internal::data::reset_color, stream); + fputs(detail::data::reset_color, stream); } template <> inline void reset_color(FILE* stream) FMT_NOEXCEPT { - fputs(internal::data::wreset_color, stream); + fputs(detail::data::wreset_color, stream); } template @@ -476,33 +476,31 @@ void vformat_to(basic_memory_buffer& buf, const text_style& ts, bool has_style = false; if (ts.has_emphasis()) { has_style = true; - auto emphasis = internal::make_emphasis(ts.get_emphasis()); + auto emphasis = detail::make_emphasis(ts.get_emphasis()); buf.append(emphasis.begin(), emphasis.end()); } if (ts.has_foreground()) { has_style = true; - auto foreground = - internal::make_foreground_color(ts.get_foreground()); + auto foreground = detail::make_foreground_color(ts.get_foreground()); buf.append(foreground.begin(), foreground.end()); } if (ts.has_background()) { has_style = true; - auto background = - internal::make_background_color(ts.get_background()); + auto background = detail::make_background_color(ts.get_background()); buf.append(background.begin(), background.end()); } - internal::vformat_to(buf, format_str, args); - if (has_style) internal::reset_color(buf); + detail::vformat_to(buf, format_str, args); + if (has_style) detail::reset_color(buf); } -} // namespace internal +} // namespace detail template > void vprint(std::FILE* f, const text_style& ts, const S& format, basic_format_args> args) { basic_memory_buffer buf; - internal::vformat_to(buf, ts, to_string_view(format), args); + detail::vformat_to(buf, ts, to_string_view(format), args); buf.push_back(Char(0)); - internal::fputs(buf.data(), f); + detail::fputs(buf.data(), f); } /** @@ -513,10 +511,10 @@ void vprint(std::FILE* f, const text_style& ts, const S& format, "Elapsed time: {0:.2f} seconds", 1.23); */ template ::value)> + FMT_ENABLE_IF(detail::is_string::value)> void print(std::FILE* f, const text_style& ts, const S& format_str, const Args&... args) { - internal::check_format_string(format_str); + detail::check_format_string(format_str); using context = buffer_context>; format_arg_store as{args...}; vprint(f, ts, format_str, basic_format_args(as)); @@ -530,7 +528,7 @@ void print(std::FILE* f, const text_style& ts, const S& format_str, "Elapsed time: {0:.2f} seconds", 1.23); */ template ::value)> + FMT_ENABLE_IF(detail::is_string::value)> void print(const text_style& ts, const S& format_str, const Args&... args) { return print(stdout, ts, format_str, args...); } @@ -540,7 +538,7 @@ inline std::basic_string vformat( const text_style& ts, const S& format_str, basic_format_args>> args) { basic_memory_buffer buf; - internal::vformat_to(buf, ts, to_string_view(format_str), args); + detail::vformat_to(buf, ts, to_string_view(format_str), args); return fmt::to_string(buf); } @@ -560,7 +558,7 @@ template > inline std::basic_string format(const text_style& ts, const S& format_str, const Args&... args) { return vformat(ts, to_string_view(format_str), - internal::make_args_checked(format_str, args...)); + detail::make_args_checked(format_str, args...)); } FMT_END_NAMESPACE diff --git a/src/fmt/compile.h b/src/fmt/compile.h index e4b12f349e..d7e6449ebb 100644 --- a/src/fmt/compile.h +++ b/src/fmt/compile.h @@ -13,7 +13,33 @@ #include "format.h" FMT_BEGIN_NAMESPACE -namespace internal { +namespace detail { + +// A compile-time string which is compiled into fast formatting code. +class compiled_string {}; + +template +struct is_compiled_string : std::is_base_of {}; + +/** + \rst + Converts a string literal *s* into a format string that will be parsed at + compile time and converted into efficient formatting code. Requires C++17 + ``constexpr if`` compiler support. + + **Example**:: + + // Converts 42 into std::string using the most efficient method and no + // runtime format string processing. + std::string s = fmt::format(FMT_COMPILE("{}"), 42); + \endrst + */ +#define FMT_COMPILE(s) FMT_STRING_IMPL(s, fmt::detail::compiled_string) + +template +const T& first(const T& value, const Tail&...) { + return value; +} // Part of a compiled format string. It can be either literal text or a // replacement field. @@ -62,13 +88,15 @@ template struct part_counter { if (begin != end) ++num_parts; } - FMT_CONSTEXPR void on_arg_id() { ++num_parts; } - FMT_CONSTEXPR void on_arg_id(int) { ++num_parts; } - FMT_CONSTEXPR void on_arg_id(basic_string_view) { ++num_parts; } + FMT_CONSTEXPR int on_arg_id() { return ++num_parts, 0; } + FMT_CONSTEXPR int on_arg_id(int) { return ++num_parts, 0; } + FMT_CONSTEXPR int on_arg_id(basic_string_view) { + return ++num_parts, 0; + } - FMT_CONSTEXPR void on_replacement_field(const Char*) {} + FMT_CONSTEXPR void on_replacement_field(int, const Char*) {} - FMT_CONSTEXPR const Char* on_format_specs(const Char* begin, + FMT_CONSTEXPR const Char* on_format_specs(int, const Char* begin, const Char* end) { // Find the matching brace. unsigned brace_counter = 0; @@ -116,25 +144,28 @@ class format_string_compiler : public error_handler { handler_(part::make_text({begin, to_unsigned(end - begin)})); } - FMT_CONSTEXPR void on_arg_id() { + FMT_CONSTEXPR int on_arg_id() { part_ = part::make_arg_index(parse_context_.next_arg_id()); + return 0; } - FMT_CONSTEXPR void on_arg_id(int id) { + FMT_CONSTEXPR int on_arg_id(int id) { parse_context_.check_arg_id(id); part_ = part::make_arg_index(id); + return 0; } - FMT_CONSTEXPR void on_arg_id(basic_string_view id) { + FMT_CONSTEXPR int on_arg_id(basic_string_view id) { part_ = part::make_arg_name(id); + return 0; } - FMT_CONSTEXPR void on_replacement_field(const Char* ptr) { + FMT_CONSTEXPR void on_replacement_field(int, const Char* ptr) { part_.arg_id_end = ptr; handler_(part_); } - FMT_CONSTEXPR const Char* on_format_specs(const Char* begin, + FMT_CONSTEXPR const Char* on_format_specs(int, const Char* begin, const Char* end) { auto repl = typename part::replacement(); dynamic_specs_handler> handler( @@ -160,23 +191,24 @@ FMT_CONSTEXPR void compile_format_string(basic_string_view format_str, format_string_compiler(format_str, handler)); } -template +template void format_arg( - basic_format_parse_context& parse_ctx, + basic_format_parse_context& parse_ctx, Context& ctx, Id arg_id) { - ctx.advance_to( - visit_format_arg(arg_formatter(ctx, &parse_ctx), ctx.arg(arg_id))); + ctx.advance_to(visit_format_arg( + arg_formatter(ctx, &parse_ctx), + ctx.arg(arg_id))); } // vformat_to is defined in a subnamespace to prevent ADL. namespace cf { -template -auto vformat_to(Range out, CompiledFormat& cf, basic_format_args args) - -> typename Context::iterator { +template +auto vformat_to(OutputIt out, CompiledFormat& cf, + basic_format_args args) -> typename Context::iterator { using char_type = typename Context::char_type; basic_format_parse_context parse_ctx( to_string_view(cf.format_str_)); - Context ctx(out.begin(), args); + Context ctx(out, args); const auto& parts = cf.parts(); for (auto part_it = std::begin(parts); part_it != std::end(parts); @@ -197,12 +229,12 @@ auto vformat_to(Range out, CompiledFormat& cf, basic_format_args args) case format_part_t::kind::arg_index: advance_to(parse_ctx, part.arg_id_end); - internal::format_arg(parse_ctx, ctx, value.arg_index); + detail::format_arg(parse_ctx, ctx, value.arg_index); break; case format_part_t::kind::arg_name: advance_to(parse_ctx, part.arg_id_end); - internal::format_arg(parse_ctx, ctx, value.str); + detail::format_arg(parse_ctx, ctx, value.str); break; case format_part_t::kind::replacement: { @@ -226,7 +258,9 @@ auto vformat_to(Range out, CompiledFormat& cf, basic_format_args args) advance_to(parse_ctx, part.arg_id_end); ctx.advance_to( - visit_format_arg(arg_formatter(ctx, nullptr, &specs), arg)); + visit_format_arg(arg_formatter( + ctx, nullptr, &specs), + arg)); break; } } @@ -240,7 +274,7 @@ struct basic_compiled_format {}; template struct compiled_format_base : basic_compiled_format { using char_type = char_t; - using parts_container = std::vector>; + using parts_container = std::vector>; parts_container compiled_parts; @@ -305,7 +339,7 @@ struct compiled_format_base::value>> const parts_container& parts() const { static FMT_CONSTEXPR_DECL const auto compiled_parts = compile_to_parts( - internal::to_string_view(S())); + detail::to_string_view(S())); return compiled_parts.data; } }; @@ -318,8 +352,8 @@ class compiled_format : private compiled_format_base { private: basic_string_view format_str_; - template - friend auto cf::vformat_to(Range out, CompiledFormat& cf, + template + friend auto cf::vformat_to(OutputIt out, CompiledFormat& cf, basic_format_args args) -> typename Context::iterator; @@ -359,8 +393,7 @@ template struct text { template OutputIt format(OutputIt out, const Args&...) const { - // TODO: reserve - return copy_str(data.begin(), data.end(), out); + return write(out, data); } }; @@ -373,33 +406,6 @@ constexpr text make_text(basic_string_view s, size_t pos, return {{&s[pos], size}}; } -template , int> = 0> -OutputIt format_default(OutputIt out, T value) { - // TODO: reserve - format_int fi(value); - return std::copy(fi.data(), fi.data() + fi.size(), out); -} - -template -OutputIt format_default(OutputIt out, double value) { - writer w(out); - w.write(value); - return w.out(); -} - -template -OutputIt format_default(OutputIt out, Char value) { - *out++ = value; - return out; -} - -template -OutputIt format_default(OutputIt out, const Char* value) { - auto length = std::char_traits::length(value); - return copy_str(value, value + length, out); -} - // A replacement field that refers to argument N. template struct field { using char_type = Char; @@ -408,13 +414,30 @@ template struct field { OutputIt format(OutputIt out, const Args&... args) const { // This ensures that the argument type is convertile to `const T&`. const T& arg = get(args...); - return format_default(out, arg); + return write(out, arg); } }; template struct is_compiled_format> : std::true_type {}; +// A replacement field that refers to argument N and has format specifiers. +template struct spec_field { + using char_type = Char; + mutable formatter fmt; + + template + OutputIt format(OutputIt out, const Args&... args) const { + // This ensures that the argument type is convertile to `const T&`. + const T& arg = get(args...); + basic_format_context ctx(out, {}); + return fmt.format(arg, ctx); + } +}; + +template +struct is_compiled_format> : std::true_type {}; + template struct concat { L lhs; R rhs; @@ -450,7 +473,8 @@ constexpr auto compile_format_string(S format_str); template constexpr auto parse_tail(T head, S format_str) { - if constexpr (POS != to_string_view(format_str).size()) { + if constexpr (POS != + basic_string_view(format_str).size()) { constexpr auto tail = compile_format_string(format_str); if constexpr (std::is_same, unknown_format>()) @@ -462,6 +486,21 @@ constexpr auto parse_tail(T head, S format_str) { } } +template struct parse_specs_result { + formatter fmt; + size_t end; +}; + +template +constexpr parse_specs_result parse_specs(basic_string_view str, + size_t pos) { + str.remove_prefix(pos); + auto ctx = basic_format_parse_context(str); + auto f = formatter(); + auto end = f.parse(ctx); + return {f, pos + (end - str.data()) + 1}; +} + // Compiles a non-empty format string and returns the compiled representation // or unknown_format() on unrecognized input. template @@ -475,12 +514,13 @@ constexpr auto compile_format_string(S format_str) { return parse_tail(make_text(str, POS, 1), format_str); } else if constexpr (str[POS + 1] == '}') { using type = get_type; - if constexpr (std::is_same::value) { - return parse_tail(field(), - format_str); - } else { - return unknown_format(); - } + return parse_tail(field(), + format_str); + } else if constexpr (str[POS + 1] == ':') { + using type = get_type; + constexpr auto result = parse_specs(str, POS + 2); + return parse_tail( + spec_field{result.fmt}, format_str); } else { return unknown_format(); } @@ -494,100 +534,130 @@ constexpr auto compile_format_string(S format_str) { format_str); } } -#endif // __cpp_if_constexpr -} // namespace internal -#if FMT_USE_CONSTEXPR -# ifdef __cpp_if_constexpr template ::value)> + FMT_ENABLE_IF(is_compile_string::value || + detail::is_compiled_string::value)> constexpr auto compile(S format_str) { constexpr basic_string_view str = format_str; if constexpr (str.size() == 0) { - return internal::make_text(str, 0, 0); + return detail::make_text(str, 0, 0); } else { constexpr auto result = - internal::compile_format_string, 0, 0>( + detail::compile_format_string, 0, 0>( format_str); if constexpr (std::is_same, - internal::unknown_format>()) { - return internal::compiled_format(to_string_view(format_str)); + detail::unknown_format>()) { + return detail::compiled_format(to_string_view(format_str)); } else { return result; } } } - -template ::value)> -std::basic_string format(const CompiledFormat& cf, const Args&... args) { - basic_memory_buffer buffer; - cf.format(std::back_inserter(buffer), args...); - return to_string(buffer); -} - -template ::value)> -OutputIt format_to(OutputIt out, const CompiledFormat& cf, - const Args&... args) { - return cf.format(out, args...); -} -# else +#else template ::value)> -constexpr auto compile(S format_str) -> internal::compiled_format { - return internal::compiled_format(to_string_view(format_str)); +constexpr auto compile(S format_str) -> detail::compiled_format { + return detail::compiled_format(to_string_view(format_str)); } -# endif // __cpp_if_constexpr -#endif // FMT_USE_CONSTEXPR +#endif // __cpp_if_constexpr // Compiles the format string which must be a string literal. template auto compile(const Char (&format_str)[N]) - -> internal::compiled_format { - return internal::compiled_format( + -> detail::compiled_format { + return detail::compiled_format( basic_string_view(format_str, N - 1)); } +} // namespace detail + +// DEPRECATED! use FMT_COMPILE instead. +template +FMT_DEPRECATED auto compile(const Args&... args) + -> decltype(detail::compile(args...)) { + return detail::compile(args...); +} + +#if FMT_USE_CONSTEXPR +# ifdef __cpp_if_constexpr template ::value)> -std::basic_string format(const CompiledFormat& cf, const Args&... args) { + FMT_ENABLE_IF(detail::is_compiled_format::value)> +FMT_INLINE std::basic_string format(const CompiledFormat& cf, + const Args&... args) { basic_memory_buffer buffer; - using range = buffer_range; - using context = buffer_context; - internal::cf::vformat_to(range(buffer), cf, - make_format_args(args...)); + detail::buffer& base = buffer; + cf.format(std::back_inserter(base), args...); return to_string(buffer); } template ::value)> +OutputIt format_to(OutputIt out, const CompiledFormat& cf, + const Args&... args) { + return cf.format(out, args...); +} +# endif // __cpp_if_constexpr +#endif // FMT_USE_CONSTEXPR + +template ::value)> +std::basic_string format(const CompiledFormat& cf, const Args&... args) { + basic_memory_buffer buffer; + using context = buffer_context; + detail::buffer& base = buffer; + detail::cf::vformat_to(std::back_inserter(base), cf, + make_format_args(args...)); + return to_string(buffer); +} + +template ::value)> +FMT_INLINE std::basic_string format(const S&, + Args&&... args) { + constexpr basic_string_view str = S(); + if (str.size() == 2 && str[0] == '{' && str[1] == '}') + return fmt::to_string(detail::first(args...)); + constexpr auto compiled = detail::compile(S()); + return format(compiled, std::forward(args)...); +} + +template ::value)> OutputIt format_to(OutputIt out, const CompiledFormat& cf, const Args&... args) { using char_type = typename CompiledFormat::char_type; - using range = internal::output_range; using context = format_context_t; - return internal::cf::vformat_to(range(out), cf, - make_format_args(args...)); + return detail::cf::vformat_to(out, cf, + make_format_args(args...)); } -template ::value)> +template ::value)> +OutputIt format_to(OutputIt out, const S&, const Args&... args) { + constexpr auto compiled = detail::compile(S()); + return format_to(out, compiled, args...); +} + +template < + typename OutputIt, typename CompiledFormat, typename... Args, + FMT_ENABLE_IF(detail::is_output_iterator::value&& std::is_base_of< + detail::basic_compiled_format, CompiledFormat>::value)> format_to_n_result format_to_n(OutputIt out, size_t n, const CompiledFormat& cf, const Args&... args) { auto it = - format_to(internal::truncating_iterator(out, n), cf, args...); + format_to(detail::truncating_iterator(out, n), cf, args...); return {it.base(), it.count()}; } template -std::size_t formatted_size(const CompiledFormat& cf, const Args&... args) { - return format_to(internal::counting_iterator(), cf, args...).count(); +size_t formatted_size(const CompiledFormat& cf, const Args&... args) { + return format_to(detail::counting_iterator(), cf, args...).count(); } FMT_END_NAMESPACE diff --git a/src/fmt/core.h b/src/fmt/core.h index 0e0824f5fe..f9155000ec 100644 --- a/src/fmt/core.h +++ b/src/fmt/core.h @@ -18,8 +18,45 @@ #include // The fmt library version in the form major * 10000 + minor * 100 + patch. -#define FMT_VERSION 60201 +#define FMT_VERSION 70002 +#ifdef __clang__ +# define FMT_CLANG_VERSION (__clang_major__ * 100 + __clang_minor__) +#else +# define FMT_CLANG_VERSION 0 +#endif + +#if defined(__GNUC__) && !defined(__clang__) +# define FMT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#else +# define FMT_GCC_VERSION 0 +#endif + +#if defined(__INTEL_COMPILER) +# define FMT_ICC_VERSION __INTEL_COMPILER +#else +# define FMT_ICC_VERSION 0 +#endif + +#if __cplusplus >= 201103L || defined(__GXX_EXPERIMENTAL_CXX0X__) +# define FMT_HAS_GXX_CXX11 FMT_GCC_VERSION +#else +# define FMT_HAS_GXX_CXX11 0 +#endif + +#ifdef __NVCC__ +# define FMT_NVCC __NVCC__ +#else +# define FMT_NVCC 0 +#endif + +#ifdef _MSC_VER +# define FMT_MSC_VER _MSC_VER +# define FMT_SUPPRESS_MSC_WARNING(n) __pragma(warning(suppress : n)) +#else +# define FMT_MSC_VER 0 +# define FMT_SUPPRESS_MSC_WARNING(n) +#endif #ifdef __has_feature # define FMT_HAS_FEATURE(x) __has_feature(x) #else @@ -27,7 +64,7 @@ #endif #if defined(__has_include) && !defined(__INTELLISENSE__) && \ - !(defined(__INTEL_COMPILER) && __INTEL_COMPILER < 1600) + !(FMT_ICC_VERSION && FMT_ICC_VERSION < 1600) # define FMT_HAS_INCLUDE(x) __has_include(x) #else # define FMT_HAS_INCLUDE(x) 0 @@ -45,43 +82,13 @@ #define FMT_HAS_CPP17_ATTRIBUTE(attribute) \ (__cplusplus >= 201703L && FMT_HAS_CPP_ATTRIBUTE(attribute)) -#ifdef __clang__ -# define FMT_CLANG_VERSION (__clang_major__ * 100 + __clang_minor__) -#else -# define FMT_CLANG_VERSION 0 -#endif - -#if defined(__GNUC__) && !defined(__clang__) -# define FMT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#else -# define FMT_GCC_VERSION 0 -#endif - -#if __cplusplus >= 201103L || defined(__GXX_EXPERIMENTAL_CXX0X__) -# define FMT_HAS_GXX_CXX11 FMT_GCC_VERSION -#else -# define FMT_HAS_GXX_CXX11 0 -#endif - -#ifdef __NVCC__ -# define FMT_NVCC __NVCC__ -#else -# define FMT_NVCC 0 -#endif - -#ifdef _MSC_VER -# define FMT_MSC_VER _MSC_VER -#else -# define FMT_MSC_VER 0 -#endif - // Check if relaxed C++14 constexpr is supported. // GCC doesn't allow throw in constexpr until version 6 (bug 67371). #ifndef FMT_USE_CONSTEXPR # define FMT_USE_CONSTEXPR \ (FMT_HAS_FEATURE(cxx_relaxed_constexpr) || FMT_MSC_VER >= 1910 || \ (FMT_GCC_VERSION >= 600 && __cplusplus >= 201402L)) && \ - !FMT_NVCC + !FMT_NVCC && !FMT_ICC_VERSION #endif #if FMT_USE_CONSTEXPR # define FMT_CONSTEXPR constexpr @@ -141,14 +148,6 @@ # define FMT_NORETURN #endif -#ifndef FMT_MAYBE_UNUSED -# if FMT_HAS_CPP17_ATTRIBUTE(maybe_unused) -# define FMT_MAYBE_UNUSED [[maybe_unused]] -# else -# define FMT_MAYBE_UNUSED -# endif -#endif - #ifndef FMT_DEPRECATED # if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VER >= 1900 # define FMT_DEPRECATED [[deprecated]] @@ -164,12 +163,20 @@ #endif // Workaround broken [[deprecated]] in the Intel, PGI and NVCC compilers. -#if defined(__INTEL_COMPILER) || defined(__PGI) || FMT_NVCC +#if FMT_ICC_VERSION || defined(__PGI) || FMT_NVCC # define FMT_DEPRECATED_ALIAS #else # define FMT_DEPRECATED_ALIAS FMT_DEPRECATED #endif +#ifndef FMT_INLINE +# if FMT_GCC_VERSION || FMT_CLANG_VERSION +# define FMT_INLINE inline __attribute__((always_inline)) +# else +# define FMT_INLINE inline +# endif +#endif + #ifndef FMT_BEGIN_NAMESPACE # if FMT_HAS_FEATURE(cxx_inline_namespaces) || FMT_GCC_VERSION >= 404 || \ FMT_MSC_VER >= 1900 @@ -181,39 +188,29 @@ # define FMT_INLINE_NAMESPACE namespace # define FMT_END_NAMESPACE \ } \ - using namespace v6; \ + using namespace v7_lmp; \ } # endif # define FMT_BEGIN_NAMESPACE \ namespace fmt { \ - FMT_INLINE_NAMESPACE v6 { + FMT_INLINE_NAMESPACE v7_lmp { #endif #if !defined(FMT_HEADER_ONLY) && defined(_WIN32) -# if FMT_MSC_VER -# define FMT_NO_W4275 __pragma(warning(suppress : 4275)) -# else -# define FMT_NO_W4275 -# endif -# define FMT_CLASS_API FMT_NO_W4275 +# define FMT_CLASS_API FMT_SUPPRESS_MSC_WARNING(4275) # ifdef FMT_EXPORT # define FMT_API __declspec(dllexport) +# define FMT_EXTERN_TEMPLATE_API FMT_API +# define FMT_EXPORTED # elif defined(FMT_SHARED) # define FMT_API __declspec(dllimport) # define FMT_EXTERN_TEMPLATE_API FMT_API # endif -#endif -#ifndef FMT_CLASS_API +#else # define FMT_CLASS_API #endif #ifndef FMT_API -# if FMT_GCC_VERSION || FMT_CLANG_VERSION -# define FMT_API __attribute__((visibility("default"))) -# define FMT_EXTERN_TEMPLATE_API FMT_API -# define FMT_INSTANTIATION_DEF_API -# else -# define FMT_API -# endif +# define FMT_API #endif #ifndef FMT_EXTERN_TEMPLATE_API # define FMT_EXTERN_TEMPLATE_API @@ -270,14 +267,11 @@ struct monostate {}; // to workaround a bug in MSVC 2019 (see #1140 and #1186). #define FMT_ENABLE_IF(...) enable_if_t<(__VA_ARGS__), int> = 0 -namespace internal { +namespace detail { // A helper function to suppress bogus "conditional expression is constant" // warnings. -template FMT_CONSTEXPR T const_check(T value) { return value; } - -// A workaround for gcc 4.8 to make void_t work in a SFINAE context. -template struct void_t_impl { using type = void; }; +template constexpr T const_check(T value) { return value; } FMT_NORETURN FMT_API void assert_fail(const char* file, int line, const char* message); @@ -290,7 +284,7 @@ FMT_NORETURN FMT_API void assert_fail(const char* file, int line, # define FMT_ASSERT(condition, message) \ ((condition) /* void() fails with -Winvalid-constexpr on clang 4.0.1 */ \ ? (void)0 \ - : ::fmt::internal::assert_fail(__FILE__, __LINE__, (message))) + : ::fmt::detail::assert_fail(__FILE__, __LINE__, (message))) # endif #endif @@ -324,7 +318,7 @@ FMT_CONSTEXPR typename std::make_unsigned::type to_unsigned(Int value) { return static_cast::type>(value); } -constexpr unsigned char micro[] = "\u00B5"; +FMT_SUPPRESS_MSC_WARNING(4566) constexpr unsigned char micro[] = "\u00B5"; template constexpr bool is_unicode() { return FMT_UNICODE || sizeof(Char) != 1 || @@ -336,10 +330,11 @@ using char8_type = char8_t; #else enum char8_type : unsigned char {}; #endif -} // namespace internal +} // namespace detail -template -using void_t = typename internal::void_t_impl::type; +#ifdef FMT_USE_INTERNAL +namespace internal = detail; // DEPRECATED +#endif /** An implementation of ``std::basic_string_view`` for pre-C++17. It provides a @@ -354,14 +349,13 @@ template class basic_string_view { size_t size_; public: - using char_type FMT_DEPRECATED_ALIAS = Char; using value_type = Char; using iterator = const Char*; - FMT_CONSTEXPR basic_string_view() FMT_NOEXCEPT : data_(nullptr), size_(0) {} + constexpr basic_string_view() FMT_NOEXCEPT : data_(nullptr), size_(0) {} /** Constructs a string reference object from a C string and a size. */ - FMT_CONSTEXPR basic_string_view(const Char* s, size_t count) FMT_NOEXCEPT + constexpr basic_string_view(const Char* s, size_t count) FMT_NOEXCEPT : data_(s), size_(count) {} @@ -384,22 +378,21 @@ template class basic_string_view { : data_(s.data()), size_(s.size()) {} - template < - typename S, - FMT_ENABLE_IF(std::is_same>::value)> + template >::value)> FMT_CONSTEXPR basic_string_view(S s) FMT_NOEXCEPT : data_(s.data()), size_(s.size()) {} /** Returns a pointer to the string data. */ - FMT_CONSTEXPR const Char* data() const { return data_; } + constexpr const Char* data() const { return data_; } /** Returns the string size. */ - FMT_CONSTEXPR size_t size() const { return size_; } + constexpr size_t size() const { return size_; } - FMT_CONSTEXPR iterator begin() const { return data_; } - FMT_CONSTEXPR iterator end() const { return data_ + size_; } + constexpr iterator begin() const { return data_; } + constexpr iterator end() const { return data_ + size_; } - FMT_CONSTEXPR const Char& operator[](size_t pos) const { return data_[pos]; } + constexpr const Char& operator[](size_t pos) const { return data_[pos]; } FMT_CONSTEXPR void remove_prefix(size_t n) { data_ += n; @@ -438,16 +431,11 @@ template class basic_string_view { using string_view = basic_string_view; using wstring_view = basic_string_view; -#ifndef __cpp_char8_t -// char8_t is deprecated; use char instead. -using char8_t FMT_DEPRECATED_ALIAS = internal::char8_type; -#endif - /** Specifies if ``T`` is a character type. Can be specialized by users. */ template struct is_char : std::false_type {}; template <> struct is_char : std::true_type {}; template <> struct is_char : std::true_type {}; -template <> struct is_char : std::true_type {}; +template <> struct is_char : std::true_type {}; template <> struct is_char : std::true_type {}; template <> struct is_char : std::true_type {}; @@ -484,14 +472,13 @@ inline basic_string_view to_string_view(basic_string_view s) { } template >::value)> -inline basic_string_view to_string_view( - internal::std_string_view s) { + FMT_ENABLE_IF(!std::is_empty>::value)> +inline basic_string_view to_string_view(detail::std_string_view s) { return s; } // A base class for compile-time strings. It is defined in the fmt namespace to -// make formatting functions visible via ADL, e.g. format(fmt("{}"), 42). +// make formatting functions visible via ADL, e.g. format(FMT_STRING("{}"), 42). struct compile_string {}; template @@ -502,9 +489,9 @@ constexpr basic_string_view to_string_view(const S& s) { return s; } -namespace internal { +namespace detail { void to_string_view(...); -using fmt::v6::to_string_view; +using fmt::v7_lmp::to_string_view; // Specifies whether S is a string type convertible to fmt::basic_string_view. // It should be a constexpr function but MSVC 2017 fails to compile it in @@ -520,16 +507,16 @@ template struct char_t_impl::value>> { }; struct error_handler { - FMT_CONSTEXPR error_handler() = default; - FMT_CONSTEXPR error_handler(const error_handler&) = default; + constexpr error_handler() = default; + constexpr error_handler(const error_handler&) = default; // This function is intentionally not constexpr to give a compile-time error. FMT_NORETURN FMT_API void on_error(const char* message); }; -} // namespace internal +} // namespace detail /** String's character type. */ -template using char_t = typename internal::char_t_impl::type; +template using char_t = typename detail::char_t_impl::type; /** \rst @@ -547,7 +534,7 @@ template using char_t = typename internal::char_t_impl::type; +-----------------------+-------------------------------------+ \endrst */ -template +template class basic_format_parse_context : private ErrorHandler { private: basic_string_view format_str_; @@ -557,26 +544,24 @@ class basic_format_parse_context : private ErrorHandler { using char_type = Char; using iterator = typename basic_string_view::iterator; - explicit FMT_CONSTEXPR basic_format_parse_context( - basic_string_view format_str, ErrorHandler eh = ErrorHandler()) + explicit constexpr basic_format_parse_context( + basic_string_view format_str, ErrorHandler eh = {}) : ErrorHandler(eh), format_str_(format_str), next_arg_id_(0) {} /** Returns an iterator to the beginning of the format string range being parsed. */ - FMT_CONSTEXPR iterator begin() const FMT_NOEXCEPT { - return format_str_.begin(); - } + constexpr iterator begin() const FMT_NOEXCEPT { return format_str_.begin(); } /** Returns an iterator past the end of the format string range being parsed. */ - FMT_CONSTEXPR iterator end() const FMT_NOEXCEPT { return format_str_.end(); } + constexpr iterator end() const FMT_NOEXCEPT { return format_str_.end(); } /** Advances the begin iterator to ``it``. */ FMT_CONSTEXPR void advance_to(iterator it) { - format_str_.remove_prefix(internal::to_unsigned(it - begin())); + format_str_.remove_prefix(detail::to_unsigned(it - begin())); } /** @@ -584,6 +569,8 @@ class basic_format_parse_context : private ErrorHandler { the next argument index and switches to the automatic indexing. */ FMT_CONSTEXPR int next_arg_id() { + // Don't check if the argument id is valid to avoid overhead and because it + // will be checked during formatting anyway. if (next_arg_id_ >= 0) return next_arg_id_++; on_error("cannot switch from manual to automatic argument indexing"); return 0; @@ -606,20 +593,15 @@ class basic_format_parse_context : private ErrorHandler { ErrorHandler::on_error(message); } - FMT_CONSTEXPR ErrorHandler error_handler() const { return *this; } + constexpr ErrorHandler error_handler() const { return *this; } }; using format_parse_context = basic_format_parse_context; using wformat_parse_context = basic_format_parse_context; -template -using basic_parse_context FMT_DEPRECATED_ALIAS = - basic_format_parse_context; -using parse_context FMT_DEPRECATED_ALIAS = basic_format_parse_context; -using wparse_context FMT_DEPRECATED_ALIAS = basic_format_parse_context; - template class basic_format_arg; template class basic_format_args; +template class dynamic_format_arg_store; // A formatter for objects of type T. template @@ -628,43 +610,44 @@ struct formatter { formatter() = delete; }; -template -struct FMT_DEPRECATED convert_to_int - : bool_constant::value && - std::is_convertible::value> {}; - // Specifies if T has an enabled formatter specialization. A type can be // formattable even if it doesn't have a formatter e.g. via a conversion. template using has_formatter = std::is_constructible>; -namespace internal { +namespace detail { -/** A contiguous memory buffer with an optional growing ability. */ +/** + \rst + A contiguous memory buffer with an optional growing ability. It is an internal + class and shouldn't be used directly, only via `~fmt::basic_memory_buffer`. + \endrst + */ template class buffer { private: T* ptr_; - std::size_t size_; - std::size_t capacity_; + size_t size_; + size_t capacity_; protected: // Don't initialize ptr_ since it is not accessed to save a few cycles. - buffer(std::size_t sz) FMT_NOEXCEPT : size_(sz), capacity_(sz) {} + FMT_SUPPRESS_MSC_WARNING(26495) + buffer(size_t sz) FMT_NOEXCEPT : size_(sz), capacity_(sz) {} - buffer(T* p = nullptr, std::size_t sz = 0, std::size_t cap = 0) FMT_NOEXCEPT + buffer(T* p = nullptr, size_t sz = 0, size_t cap = 0) FMT_NOEXCEPT : ptr_(p), size_(sz), capacity_(cap) {} /** Sets the buffer data and capacity. */ - void set(T* buf_data, std::size_t buf_capacity) FMT_NOEXCEPT { + void set(T* buf_data, size_t buf_capacity) FMT_NOEXCEPT { ptr_ = buf_data; capacity_ = buf_capacity; } /** Increases the buffer capacity to hold at least *capacity* elements. */ - virtual void grow(std::size_t capacity) = 0; + virtual void grow(size_t capacity) = 0; public: using value_type = T; @@ -681,10 +664,10 @@ template class buffer { const T* end() const FMT_NOEXCEPT { return ptr_ + size_; } /** Returns the size of this buffer. */ - std::size_t size() const FMT_NOEXCEPT { return size_; } + size_t size() const FMT_NOEXCEPT { return size_; } /** Returns the capacity of this buffer. */ - std::size_t capacity() const FMT_NOEXCEPT { return capacity_; } + size_t capacity() const FMT_NOEXCEPT { return capacity_; } /** Returns a pointer to the buffer data. */ T* data() FMT_NOEXCEPT { return ptr_; } @@ -695,7 +678,7 @@ template class buffer { /** Resizes the buffer. If T is a POD type new elements may not be initialized. */ - void resize(std::size_t new_size) { + void resize(size_t new_size) { reserve(new_size); size_ = new_size; } @@ -704,7 +687,7 @@ template class buffer { void clear() { size_ = 0; } /** Reserves space to store at least *capacity* elements. */ - void reserve(std::size_t new_capacity) { + void reserve(size_t new_capacity) { if (new_capacity > capacity_) grow(new_capacity); } @@ -729,7 +712,7 @@ class container_buffer : public buffer { Container& container_; protected: - void grow(std::size_t capacity) FMT_OVERRIDE { + void grow(size_t capacity) FMT_OVERRIDE { container_.resize(capacity); this->set(&container_[0], capacity); } @@ -760,12 +743,78 @@ template using has_fallback_formatter = std::is_constructible>; -template struct named_arg_base; -template struct named_arg; +struct view {}; + +template struct named_arg : view { + const Char* name; + const T& value; + named_arg(const Char* n, const T& v) : name(n), value(v) {} +}; + +template struct named_arg_info { + const Char* name; + int id; +}; + +template +struct arg_data { + // args_[0].named_args points to named_args_ to avoid bloating format_args. + T args_[1 + (NUM_ARGS != 0 ? NUM_ARGS : 1)]; + named_arg_info named_args_[NUM_NAMED_ARGS]; + + template + arg_data(const U&... init) : args_{T(named_args_, NUM_NAMED_ARGS), init...} {} + arg_data(const arg_data& other) = delete; + const T* args() const { return args_ + 1; } + named_arg_info* named_args() { return named_args_; } +}; + +template +struct arg_data { + T args_[NUM_ARGS != 0 ? NUM_ARGS : 1]; + + template + FMT_INLINE arg_data(const U&... init) : args_{init...} {} + FMT_INLINE const T* args() const { return args_; } + FMT_INLINE std::nullptr_t named_args() { return nullptr; } +}; + +template +inline void init_named_args(named_arg_info*, int, int) {} + +template +void init_named_args(named_arg_info* named_args, int arg_count, + int named_arg_count, const T&, const Tail&... args) { + init_named_args(named_args, arg_count + 1, named_arg_count, args...); +} + +template +void init_named_args(named_arg_info* named_args, int arg_count, + int named_arg_count, const named_arg& arg, + const Tail&... args) { + named_args[named_arg_count++] = {arg.name, arg_count}; + init_named_args(named_args, arg_count + 1, named_arg_count, args...); +} + +template +FMT_INLINE void init_named_args(std::nullptr_t, int, int, const Args&...) {} + +template struct is_named_arg : std::false_type {}; + +template +struct is_named_arg> : std::true_type {}; + +template constexpr size_t count() { return B ? 1 : 0; } +template constexpr size_t count() { + return (B1 ? 1 : 0) + count(); +} + +template constexpr size_t count_named_args() { + return count::value...>(); +} enum class type { none_type, - named_arg_type, // Integer types should go first, int_type, uint_type, @@ -796,7 +845,6 @@ struct type_constant : std::integral_constant {}; struct type_constant \ : std::integral_constant {} -FMT_TYPE_CONSTANT(const named_arg_base&, named_arg_type); FMT_TYPE_CONSTANT(int, int_type); FMT_TYPE_CONSTANT(unsigned, uint_type); FMT_TYPE_CONSTANT(long long, long_long_type); @@ -812,26 +860,28 @@ FMT_TYPE_CONSTANT(const Char*, cstring_type); FMT_TYPE_CONSTANT(basic_string_view, string_type); FMT_TYPE_CONSTANT(const void*, pointer_type); -FMT_CONSTEXPR bool is_integral_type(type t) { - FMT_ASSERT(t != type::named_arg_type, "invalid argument type"); +constexpr bool is_integral_type(type t) { return t > type::none_type && t <= type::last_integer_type; } -FMT_CONSTEXPR bool is_arithmetic_type(type t) { - FMT_ASSERT(t != type::named_arg_type, "invalid argument type"); +constexpr bool is_arithmetic_type(type t) { return t > type::none_type && t <= type::last_numeric_type; } template struct string_value { const Char* data; - std::size_t size; + size_t size; +}; + +template struct named_arg_value { + const named_arg_info* data; + size_t size; }; template struct custom_value { - using parse_context = basic_format_parse_context; + using parse_context = typename Context::parse_context_type; const void* value; - void (*format)(const void* arg, - typename Context::parse_context_type& parse_ctx, Context& ctx); + void (*format)(const void* arg, parse_context& parse_ctx, Context& ctx); }; // A formatting argument value. @@ -854,28 +904,30 @@ template class value { const void* pointer; string_value string; custom_value custom; - const named_arg_base* named_arg; + named_arg_value named_args; }; - FMT_CONSTEXPR value(int val = 0) : int_value(val) {} - FMT_CONSTEXPR value(unsigned val) : uint_value(val) {} - value(long long val) : long_long_value(val) {} - value(unsigned long long val) : ulong_long_value(val) {} - value(int128_t val) : int128_value(val) {} - value(uint128_t val) : uint128_value(val) {} - value(float val) : float_value(val) {} - value(double val) : double_value(val) {} - value(long double val) : long_double_value(val) {} - value(bool val) : bool_value(val) {} - value(char_type val) : char_value(val) {} - value(const char_type* val) { string.data = val; } - value(basic_string_view val) { + constexpr FMT_INLINE value(int val = 0) : int_value(val) {} + constexpr FMT_INLINE value(unsigned val) : uint_value(val) {} + FMT_INLINE value(long long val) : long_long_value(val) {} + FMT_INLINE value(unsigned long long val) : ulong_long_value(val) {} + FMT_INLINE value(int128_t val) : int128_value(val) {} + FMT_INLINE value(uint128_t val) : uint128_value(val) {} + FMT_INLINE value(float val) : float_value(val) {} + FMT_INLINE value(double val) : double_value(val) {} + FMT_INLINE value(long double val) : long_double_value(val) {} + FMT_INLINE value(bool val) : bool_value(val) {} + FMT_INLINE value(char_type val) : char_value(val) {} + FMT_INLINE value(const char_type* val) { string.data = val; } + FMT_INLINE value(basic_string_view val) { string.data = val.data(); string.size = val.size(); } - value(const void* val) : pointer(val) {} + FMT_INLINE value(const void* val) : pointer(val) {} + FMT_INLINE value(const named_arg_info* args, size_t size) + : named_args{args, size} {} - template value(const T& val) { + template FMT_INLINE value(const T& val) { custom.value = &val; // Get the formatter type through the context to allow different contexts // have different extension points, e.g. `formatter` for `format` and @@ -886,8 +938,6 @@ template class value { fallback_formatter>>; } - value(const named_arg_base& val) { named_arg = &val; } - private: // Formats an argument of a custom type, such as a user-defined class. template @@ -973,6 +1023,14 @@ template struct arg_mapper { static_assert(std::is_same::value, "invalid string type"); return reinterpret_cast(val); } + FMT_CONSTEXPR const char* map(signed char* val) { + const auto* const_val = val; + return map(const_val); + } + FMT_CONSTEXPR const char* map(unsigned char* val) { + const auto* const_val = val; + return map(const_val); + } FMT_CONSTEXPR const void* map(void* val) { return val; } FMT_CONSTEXPR const void* map(const void* val) { return val; } @@ -1004,11 +1062,9 @@ template struct arg_mapper { } template - FMT_CONSTEXPR const named_arg_base& map( - const named_arg& val) { - auto arg = make_arg(val.value); - std::memcpy(val.data, &arg, sizeof(arg)); - return val; + FMT_CONSTEXPR auto map(const named_arg& val) + -> decltype(std::declval().map(val.value)) { + return map(val.value); } int map(...) { @@ -1028,23 +1084,22 @@ using mapped_type_constant = type_constant().map(std::declval())), typename Context::char_type>; -enum { packed_arg_bits = 5 }; +enum { packed_arg_bits = 4 }; // Maximum number of arguments with packed types. -enum { max_packed_args = 63 / packed_arg_bits }; +enum { max_packed_args = 62 / packed_arg_bits }; enum : unsigned long long { is_unpacked_bit = 1ULL << 63 }; - -template class arg_map; -} // namespace internal +enum : unsigned long long { has_named_args_bit = 1ULL << 62 }; +} // namespace detail // A formatting argument. It is a trivially copyable/constructible type to // allow storage in basic_memory_buffer. template class basic_format_arg { private: - internal::value value_; - internal::type type_; + detail::value value_; + detail::type type_; template - friend FMT_CONSTEXPR basic_format_arg internal::make_arg( + friend FMT_CONSTEXPR basic_format_arg detail::make_arg( const T& value); template @@ -1053,14 +1108,20 @@ template class basic_format_arg { -> decltype(vis(0)); friend class basic_format_args; - friend class internal::arg_map; + friend class dynamic_format_arg_store; using char_type = typename Context::char_type; + template + friend struct detail::arg_data; + + basic_format_arg(const detail::named_arg_info* args, size_t size) + : value_(args, size) {} + public: class handle { public: - explicit handle(internal::custom_value custom) : custom_(custom) {} + explicit handle(detail::custom_value custom) : custom_(custom) {} void format(typename Context::parse_context_type& parse_ctx, Context& ctx) const { @@ -1068,19 +1129,19 @@ template class basic_format_arg { } private: - internal::custom_value custom_; + detail::custom_value custom_; }; - FMT_CONSTEXPR basic_format_arg() : type_(internal::type::none_type) {} + constexpr basic_format_arg() : type_(detail::type::none_type) {} - FMT_CONSTEXPR explicit operator bool() const FMT_NOEXCEPT { - return type_ != internal::type::none_type; + constexpr explicit operator bool() const FMT_NOEXCEPT { + return type_ != detail::type::none_type; } - internal::type type() const { return type_; } + detail::type type() const { return type_; } - bool is_integral() const { return internal::is_integral_type(type_); } - bool is_arithmetic() const { return internal::is_arithmetic_type(type_); } + bool is_integral() const { return detail::is_integral_type(type_); } + bool is_arithmetic() const { return detail::is_arithmetic_type(type_); } }; /** @@ -1091,92 +1152,73 @@ template class basic_format_arg { \endrst */ template -FMT_CONSTEXPR auto visit_format_arg(Visitor&& vis, - const basic_format_arg& arg) - -> decltype(vis(0)) { +FMT_CONSTEXPR_DECL FMT_INLINE auto visit_format_arg( + Visitor&& vis, const basic_format_arg& arg) -> decltype(vis(0)) { using char_type = typename Context::char_type; switch (arg.type_) { - case internal::type::none_type: + case detail::type::none_type: break; - case internal::type::named_arg_type: - FMT_ASSERT(false, "invalid argument type"); - break; - case internal::type::int_type: + case detail::type::int_type: return vis(arg.value_.int_value); - case internal::type::uint_type: + case detail::type::uint_type: return vis(arg.value_.uint_value); - case internal::type::long_long_type: + case detail::type::long_long_type: return vis(arg.value_.long_long_value); - case internal::type::ulong_long_type: + case detail::type::ulong_long_type: return vis(arg.value_.ulong_long_value); #if FMT_USE_INT128 - case internal::type::int128_type: + case detail::type::int128_type: return vis(arg.value_.int128_value); - case internal::type::uint128_type: + case detail::type::uint128_type: return vis(arg.value_.uint128_value); #else - case internal::type::int128_type: - case internal::type::uint128_type: + case detail::type::int128_type: + case detail::type::uint128_type: break; #endif - case internal::type::bool_type: + case detail::type::bool_type: return vis(arg.value_.bool_value); - case internal::type::char_type: + case detail::type::char_type: return vis(arg.value_.char_value); - case internal::type::float_type: + case detail::type::float_type: return vis(arg.value_.float_value); - case internal::type::double_type: + case detail::type::double_type: return vis(arg.value_.double_value); - case internal::type::long_double_type: + case detail::type::long_double_type: return vis(arg.value_.long_double_value); - case internal::type::cstring_type: + case detail::type::cstring_type: return vis(arg.value_.string.data); - case internal::type::string_type: + case detail::type::string_type: return vis(basic_string_view(arg.value_.string.data, arg.value_.string.size)); - case internal::type::pointer_type: + case detail::type::pointer_type: return vis(arg.value_.pointer); - case internal::type::custom_type: + case detail::type::custom_type: return vis(typename basic_format_arg::handle(arg.value_.custom)); } return vis(monostate()); } -namespace internal { -// A map from argument names to their values for named arguments. -template class arg_map { - private: - using char_type = typename Context::char_type; +// Checks whether T is a container with contiguous storage. +template struct is_contiguous : std::false_type {}; +template +struct is_contiguous> : std::true_type {}; +template +struct is_contiguous> : std::true_type {}; - struct entry { - basic_string_view name; - basic_format_arg arg; - }; +namespace detail { - entry* map_; - unsigned size_; +template +struct is_back_insert_iterator : std::false_type {}; +template +struct is_back_insert_iterator> + : std::true_type {}; - void push_back(value val) { - const auto& named = *val.named_arg; - map_[size_] = {named.name, named.template deserialize()}; - ++size_; - } - - public: - arg_map(const arg_map&) = delete; - void operator=(const arg_map&) = delete; - arg_map() : map_(nullptr), size_(0) {} - void init(const basic_format_args& args); - ~arg_map() { delete[] map_; } - - basic_format_arg find(basic_string_view name) const { - // The list is unsorted, so just return the first matching name. - for (entry *it = map_, *end = map_ + size_; it != end; ++it) { - if (it->name == name) return it->arg; - } - return {}; - } -}; +template +struct is_contiguous_back_insert_iterator : std::false_type {}; +template +struct is_contiguous_back_insert_iterator> + : is_contiguous {}; // A type-erased reference to an std::locale to avoid heavy include. class locale_ref { @@ -1224,10 +1266,14 @@ inline basic_format_arg make_arg(const T& value) { } template struct is_reference_wrapper : std::false_type {}; - template struct is_reference_wrapper> : std::true_type {}; +template const T& unwrap(const T& v) { return v; } +template const T& unwrap(const std::reference_wrapper& v) { + return static_cast(v); +} + class dynamic_arg_list { // Workaround for clang's -Wweak-vtables. Unlike for regular classes, for // templates it doesn't complain about inability to deduce single translation @@ -1252,14 +1298,14 @@ class dynamic_arg_list { public: template const T& push(const Arg& arg) { - auto node = std::unique_ptr>(new typed_node(arg)); - auto& value = node->value; - node->next = std::move(head_); - head_ = std::move(node); + auto new_node = std::unique_ptr>(new typed_node(arg)); + auto& value = new_node->value; + new_node->next = std::move(head_); + head_ = std::move(new_node); return value; } }; -} // namespace internal +} // namespace detail // Formatting context. template class basic_format_context { @@ -1270,8 +1316,7 @@ template class basic_format_context { private: OutputIt out_; basic_format_args args_; - internal::arg_map map_; - internal::locale_ref loc_; + detail::locale_ref loc_; public: using iterator = OutputIt; @@ -1287,34 +1332,38 @@ template class basic_format_context { */ basic_format_context(OutputIt out, basic_format_args ctx_args, - internal::locale_ref loc = internal::locale_ref()) + detail::locale_ref loc = detail::locale_ref()) : out_(out), args_(ctx_args), loc_(loc) {} format_arg arg(int id) const { return args_.get(id); } + format_arg arg(basic_string_view name) { return args_.get(name); } + int arg_id(basic_string_view name) { return args_.get_id(name); } + const basic_format_args& args() const { return args_; } - // Checks if manual indexing is used and returns the argument with the - // specified name. - format_arg arg(basic_string_view name); - - internal::error_handler error_handler() { return {}; } + detail::error_handler error_handler() { return {}; } void on_error(const char* message) { error_handler().on_error(message); } // Returns an iterator to the beginning of the output range. iterator out() { return out_; } // Advances the begin iterator to ``it``. - void advance_to(iterator it) { out_ = it; } + void advance_to(iterator it) { + if (!detail::is_back_insert_iterator()) out_ = it; + } - internal::locale_ref locale() { return loc_; } + detail::locale_ref locale() { return loc_; } }; template using buffer_context = - basic_format_context>, - Char>; + basic_format_context>, Char>; using format_context = buffer_context; using wformat_context = buffer_context; +// Workaround a bug in gcc: https://stackoverflow.com/q/62767544/471164. +#define FMT_BUFFER_CONTEXT(Char) \ + basic_format_context>, Char> + /** \rst An array of references to arguments. It can be implicitly converted into @@ -1331,29 +1380,35 @@ class format_arg_store { private: static const size_t num_args = sizeof...(Args); - static const bool is_packed = num_args < internal::max_packed_args; + static const size_t num_named_args = detail::count_named_args(); + static const bool is_packed = num_args <= detail::max_packed_args; - using value_type = conditional_t, + using value_type = conditional_t, basic_format_arg>; - // If the arguments are not packed, add one more element to mark the end. - value_type data_[num_args + (num_args == 0 ? 1 : 0)]; + detail::arg_data + data_; friend class basic_format_args; - public: - static constexpr unsigned long long types = - is_packed ? internal::encode_types() - : internal::is_unpacked_bit | num_args; + static constexpr unsigned long long desc = + (is_packed ? detail::encode_types() + : detail::is_unpacked_bit | num_args) | + (num_named_args != 0 + ? static_cast(detail::has_named_args_bit) + : 0); + public: format_arg_store(const Args&... args) : #if FMT_GCC_VERSION && FMT_GCC_VERSION < 409 basic_format_args(*this), #endif - data_{internal::make_arg< + data_{detail::make_arg< is_packed, Context, - internal::mapped_type_constant::value>(args)...} { + detail::mapped_type_constant::value>(args)...} { + detail::init_named_args(data_.named_args(), 0, 0, args...); } }; @@ -1373,8 +1428,24 @@ inline format_arg_store make_format_args( /** \rst - A dynamic version of `fmt::format_arg_store<>`. - It's equipped with a storage to potentially temporary objects which lifetime + Returns a named argument to be used in a formatting function. It should only + be used in a call to a formatting function. + + **Example**:: + + fmt::print("Elapsed time: {s:.2f} seconds", fmt::arg("s", 1.23)); + \endrst + */ +template +inline detail::named_arg arg(const Char* name, const T& arg) { + static_assert(!detail::is_named_arg(), "nested named arguments"); + return {name, arg}; +} + +/** + \rst + A dynamic version of `fmt::format_arg_store`. + It's equipped with a storage to potentially temporary objects which lifetimes could be shorter than the format arguments object. It can be implicitly converted into `~fmt::basic_format_args` for passing @@ -1392,49 +1463,73 @@ class dynamic_format_arg_store using char_type = typename Context::char_type; template struct need_copy { - static constexpr internal::type mapped_type = - internal::mapped_type_constant::value; + static constexpr detail::type mapped_type = + detail::mapped_type_constant::value; enum { - value = !(internal::is_reference_wrapper::value || + value = !(detail::is_reference_wrapper::value || std::is_same>::value || - std::is_same>::value || - (mapped_type != internal::type::cstring_type && - mapped_type != internal::type::string_type && - mapped_type != internal::type::custom_type && - mapped_type != internal::type::named_arg_type)) + std::is_same>::value || + (mapped_type != detail::type::cstring_type && + mapped_type != detail::type::string_type && + mapped_type != detail::type::custom_type)) }; }; template - using stored_type = conditional_t::value, + using stored_type = conditional_t::value, std::basic_string, T>; // Storage of basic_format_arg must be contiguous. std::vector> data_; + std::vector> named_info_; // Storage of arguments not fitting into basic_format_arg must grow // without relocation because items in data_ refer to it. - internal::dynamic_arg_list dynamic_args_; + detail::dynamic_arg_list dynamic_args_; friend class basic_format_args; unsigned long long get_types() const { - return internal::is_unpacked_bit | data_.size(); + return detail::is_unpacked_bit | data_.size() | + (named_info_.empty() + ? 0ULL + : static_cast(detail::has_named_args_bit)); + } + + const basic_format_arg* data() const { + return named_info_.empty() ? data_.data() : data_.data() + 1; } template void emplace_arg(const T& arg) { - data_.emplace_back(internal::make_arg(arg)); + data_.emplace_back(detail::make_arg(arg)); + } + + template + void emplace_arg(const detail::named_arg& arg) { + if (named_info_.empty()) { + constexpr const detail::named_arg_info* zero_ptr{nullptr}; + data_.insert(data_.begin(), {zero_ptr, 0}); + } + data_.emplace_back(detail::make_arg(detail::unwrap(arg.value))); + auto pop_one = [](std::vector>* data) { + data->pop_back(); + }; + std::unique_ptr>, decltype(pop_one)> + guard{&data_, pop_one}; + named_info_.push_back({arg.name, static_cast(data_.size() - 2u)}); + data_[0].value_.named_args = {named_info_.data(), named_info_.size()}; + guard.release(); } public: /** \rst - Adds an argument into the dynamic store for later passing to a formating + Adds an argument into the dynamic store for later passing to a formatting function. - Note that custom types and string types (but not string views!) are copied - into the store with dynamic memory (in addition to resizing vector). + Note that custom types and string types (but not string views) are copied + into the store dynamically allocating memory if necessary. **Example**:: @@ -1446,25 +1541,78 @@ class dynamic_format_arg_store \endrst */ template void push_back(const T& arg) { - static_assert( - !std::is_base_of, T>::value, - "named arguments are not supported yet"); - if (internal::const_check(need_copy::value)) + if (detail::const_check(need_copy::value)) emplace_arg(dynamic_args_.push>(arg)); else - emplace_arg(arg); + emplace_arg(detail::unwrap(arg)); } /** + \rst Adds a reference to the argument into the dynamic store for later passing to - a formating function. + a formatting function. Supports named arguments wrapped in + ``std::reference_wrapper`` via ``std::ref()``/``std::cref()``. + + **Example**:: + + fmt::dynamic_format_arg_store store; + char str[] = "1234567890"; + store.push_back(std::cref(str)); + int a1_val{42}; + auto a1 = fmt::arg("a1_", a1_val); + store.push_back(std::cref(a1)); + + // Changing str affects the output but only for string and custom types. + str[0] = 'X'; + + std::string result = fmt::vformat("{} and {a1_}"); + assert(result == "X234567890 and 42"); + \endrst */ template void push_back(std::reference_wrapper arg) { static_assert( - need_copy::value, + detail::is_named_arg::type>::value || + need_copy::value, "objects of built-in types and string views are always copied"); emplace_arg(arg.get()); } + + /** + Adds named argument into the dynamic store for later passing to a formatting + function. ``std::reference_wrapper`` is supported to avoid copying of the + argument. + */ + template + void push_back(const detail::named_arg& arg) { + const char_type* arg_name = + dynamic_args_.push>(arg.name).c_str(); + if (detail::const_check(need_copy::value)) { + emplace_arg( + fmt::arg(arg_name, dynamic_args_.push>(arg.value))); + } else { + emplace_arg(fmt::arg(arg_name, arg.value)); + } + } + + /** Erase all elements from the store */ + void clear() { + data_.clear(); + named_info_.clear(); + dynamic_args_ = detail::dynamic_arg_list(); + } + + /** + \rst + Reserves space to store at least *new_cap* arguments including + *new_cap_named* named arguments. + \endrst + */ + void reserve(size_t new_cap, size_t new_cap_named) { + FMT_ASSERT(new_cap >= new_cap_named, + "Set of arguments includes set of named arguments"); + data_.reserve(new_cap); + named_info_.reserve(new_cap_named); + } }; /** @@ -1483,49 +1631,40 @@ template class basic_format_args { using format_arg = basic_format_arg; private: - // To reduce compiled code size per formatting function call, types of first - // max_packed_args arguments are passed in the types_ field. - unsigned long long types_; + // A descriptor that contains information about formatting arguments. + // If the number of arguments is less or equal to max_packed_args then + // argument types are passed in the descriptor. This reduces binary code size + // per formatting function call. + unsigned long long desc_; union { - // If the number of arguments is less than max_packed_args, the argument - // values are stored in values_, otherwise they are stored in args_. - // This is done to reduce compiled code size as storing larger objects + // If is_packed() returns true then argument values are stored in values_; + // otherwise they are stored in args_. This is done to improve cache + // locality and reduce compiled code size since storing larger objects // may require more code (at least on x86-64) even if the same amount of // data is actually copied to stack. It saves ~10% on the bloat test. - const internal::value* values_; + const detail::value* values_; const format_arg* args_; }; - bool is_packed() const { return (types_ & internal::is_unpacked_bit) == 0; } - - internal::type type(int index) const { - int shift = index * internal::packed_arg_bits; - unsigned int mask = (1 << internal::packed_arg_bits) - 1; - return static_cast((types_ >> shift) & mask); + bool is_packed() const { return (desc_ & detail::is_unpacked_bit) == 0; } + bool has_named_args() const { + return (desc_ & detail::has_named_args_bit) != 0; } - friend class internal::arg_map; - - void set_data(const internal::value* values) { values_ = values; } - void set_data(const format_arg* args) { args_ = args; } - - format_arg do_get(int index) const { - format_arg arg; - if (!is_packed()) { - auto num_args = max_size(); - if (index < num_args) arg = args_[index]; - return arg; - } - if (index > internal::max_packed_args) return arg; - arg.type_ = type(index); - if (arg.type_ == internal::type::none_type) return arg; - internal::value& val = arg.value_; - val = values_[index]; - return arg; + detail::type type(int index) const { + int shift = index * detail::packed_arg_bits; + unsigned int mask = (1 << detail::packed_arg_bits) - 1; + return static_cast((desc_ >> shift) & mask); } + basic_format_args(unsigned long long desc, + const detail::value* values) + : desc_(desc), values_(values) {} + basic_format_args(unsigned long long desc, const format_arg* args) + : desc_(desc), args_(args) {} + public: - basic_format_args() : types_(0) {} + basic_format_args() : desc_(0) {} /** \rst @@ -1533,10 +1672,8 @@ template class basic_format_args { \endrst */ template - basic_format_args(const format_arg_store& store) - : types_(store.types) { - set_data(store.data_); - } + FMT_INLINE basic_format_args(const format_arg_store& store) + : basic_format_args(store.desc, store.data_.args()) {} /** \rst @@ -1544,10 +1681,8 @@ template class basic_format_args { `~fmt::dynamic_format_arg_store`. \endrst */ - basic_format_args(const dynamic_format_arg_store& store) - : types_(store.get_types()) { - set_data(store.data_.data()); - } + FMT_INLINE basic_format_args(const dynamic_format_arg_store& store) + : basic_format_args(store.get_types(), store.data()) {} /** \rst @@ -1555,22 +1690,42 @@ template class basic_format_args { \endrst */ basic_format_args(const format_arg* args, int count) - : types_(internal::is_unpacked_bit | internal::to_unsigned(count)) { - set_data(args); - } + : basic_format_args(detail::is_unpacked_bit | detail::to_unsigned(count), + args) {} - /** Returns the argument at specified index. */ - format_arg get(int index) const { - format_arg arg = do_get(index); - if (arg.type_ == internal::type::named_arg_type) - arg = arg.value_.named_arg->template deserialize(); + /** Returns the argument with the specified id. */ + format_arg get(int id) const { + format_arg arg; + if (!is_packed()) { + if (id < max_size()) arg = args_[id]; + return arg; + } + if (id >= detail::max_packed_args) return arg; + arg.type_ = type(id); + if (arg.type_ == detail::type::none_type) return arg; + arg.value_ = values_[id]; return arg; } + template format_arg get(basic_string_view name) const { + int id = get_id(name); + return id >= 0 ? get(id) : format_arg(); + } + + template int get_id(basic_string_view name) const { + if (!has_named_args()) return {}; + const auto& named_args = + (is_packed() ? values_[-1] : args_[-1].value_).named_args; + for (size_t i = 0; i < named_args.size; ++i) { + if (named_args.data[i].name == name) return named_args.data[i].id; + } + return -1; + } + int max_size() const { - unsigned long long max_packed = internal::max_packed_args; + unsigned long long max_packed = detail::max_packed_args; return static_cast(is_packed() ? max_packed - : types_ & ~internal::is_unpacked_bit); + : desc_ & ~detail::is_unpacked_bit); } }; @@ -1578,93 +1733,48 @@ template class basic_format_args { // It is a separate type rather than an alias to make symbols readable. struct format_args : basic_format_args { template - format_args(Args&&... args) - : basic_format_args(static_cast(args)...) {} + FMT_INLINE format_args(const Args&... args) : basic_format_args(args...) {} }; struct wformat_args : basic_format_args { - template - wformat_args(Args&&... args) - : basic_format_args(static_cast(args)...) {} + using basic_format_args::basic_format_args; }; -template struct is_contiguous : std::false_type {}; - -template -struct is_contiguous> : std::true_type {}; - -template -struct is_contiguous> : std::true_type {}; - -namespace internal { - -template -struct is_contiguous_back_insert_iterator : std::false_type {}; -template -struct is_contiguous_back_insert_iterator> - : is_contiguous {}; - -template struct named_arg_base { - basic_string_view name; - - // Serialized value. - mutable char data[sizeof(basic_format_arg>)]; - - named_arg_base(basic_string_view nm) : name(nm) {} - - template basic_format_arg deserialize() const { - basic_format_arg arg; - std::memcpy(&arg, data, sizeof(basic_format_arg)); - return arg; - } -}; - -struct view {}; - -template -struct named_arg : view, named_arg_base { - const T& value; - - named_arg(basic_string_view name, const T& val) - : named_arg_base(name), value(val) {} -}; +namespace detail { +// Reports a compile-time error if S is not a valid format string. template ::value)> -inline void check_format_string(const S&) { -#if defined(FMT_ENFORCE_COMPILE_STRING) +FMT_INLINE void check_format_string(const S&) { +#ifdef FMT_ENFORCE_COMPILE_STRING static_assert(is_compile_string::value, - "FMT_ENFORCE_COMPILE_STRING requires all format strings to " - "utilize FMT_STRING() or fmt()."); + "FMT_ENFORCE_COMPILE_STRING requires all format strings to use " + "FMT_STRING."); #endif } template ::value)> void check_format_string(S); -template struct bool_pack; -template -using all_true = - std::is_same, bool_pack>; - template > inline format_arg_store, remove_reference_t...> make_args_checked(const S& format_str, const remove_reference_t&... args) { - static_assert( - all_true<(!std::is_base_of>::value || - !std::is_reference::value)...>::value, - "passing views as lvalues is disallowed"); + static_assert(count<(std::is_base_of>::value && + std::is_reference::value)...>() == 0, + "passing views as lvalues is disallowed"); check_format_string(format_str); return {args...}; } -template +template ::value)> std::basic_string vformat( basic_string_view format_str, basic_format_args>> args); +FMT_API std::string vformat(string_view format_str, format_args args); + template -typename buffer_context::iterator vformat_to( +typename FMT_BUFFER_CONTEXT(Char)::iterator vformat_to( buffer& buf, basic_string_view format_str, - basic_format_args>> args); + basic_format_args)> args); template ::value)> @@ -1674,58 +1784,38 @@ FMT_API void vprint_mojibake(std::FILE*, string_view, format_args); #ifndef _WIN32 inline void vprint_mojibake(std::FILE*, string_view, format_args) {} #endif -} // namespace internal - -/** - \rst - Returns a named argument to be used in a formatting function. It should only - be used in a call to a formatting function. - - **Example**:: - - fmt::print("Elapsed time: {s:.2f} seconds", fmt::arg("s", 1.23)); - \endrst - */ -template > -inline internal::named_arg arg(const S& name, const T& arg) { - static_assert(internal::is_string::value, ""); - return {name, arg}; -} - -// Disable nested named arguments, e.g. ``arg("a", arg("b", 42))``. -template -void arg(S, internal::named_arg) = delete; +} // namespace detail /** Formats a string and writes the output to ``out``. */ // GCC 8 and earlier cannot handle std::back_insert_iterator with // vformat_to(...) overload, so SFINAE on iterator type instead. -template , - FMT_ENABLE_IF( - internal::is_contiguous_back_insert_iterator::value)> +template < + typename OutputIt, typename S, typename Char = char_t, + FMT_ENABLE_IF(detail::is_contiguous_back_insert_iterator::value)> OutputIt vformat_to( OutputIt out, const S& format_str, basic_format_args>> args) { - using container = remove_reference_t; - internal::container_buffer buf((internal::get_container(out))); - internal::vformat_to(buf, to_string_view(format_str), args); + auto& c = detail::get_container(out); + detail::container_buffer> buf(c); + detail::vformat_to(buf, to_string_view(format_str), args); return out; } template ::value&& internal::is_string::value)> + is_contiguous::value&& detail::is_string::value)> inline std::back_insert_iterator format_to( std::back_insert_iterator out, const S& format_str, Args&&... args) { return vformat_to(out, to_string_view(format_str), - internal::make_args_checked(format_str, args...)); + detail::make_args_checked(format_str, args...)); } template > -inline std::basic_string vformat( +FMT_INLINE std::basic_string vformat( const S& format_str, basic_format_args>> args) { - return internal::vformat(to_string_view(format_str), args); + return detail::vformat(to_string_view(format_str), args); } /** @@ -1741,10 +1831,9 @@ inline std::basic_string vformat( // Pass char_t as a default template parameter instead of using // std::basic_string> to reduce the symbol size. template > -inline std::basic_string format(const S& format_str, Args&&... args) { - return internal::vformat( - to_string_view(format_str), - internal::make_args_checked(format_str, args...)); +FMT_INLINE std::basic_string format(const S& format_str, Args&&... args) { + const auto& vargs = detail::make_args_checked(format_str, args...); + return detail::vformat(to_string_view(format_str), vargs); } FMT_API void vprint(string_view, format_args); @@ -1763,12 +1852,10 @@ FMT_API void vprint(std::FILE*, string_view, format_args); */ template > inline void print(std::FILE* f, const S& format_str, Args&&... args) { - return internal::is_unicode() - ? vprint(f, to_string_view(format_str), - internal::make_args_checked(format_str, args...)) - : internal::vprint_mojibake( - f, to_string_view(format_str), - internal::make_args_checked(format_str, args...)); + const auto& vargs = detail::make_args_checked(format_str, args...); + return detail::is_unicode() + ? vprint(f, to_string_view(format_str), vargs) + : detail::vprint_mojibake(f, to_string_view(format_str), vargs); } /** @@ -1784,12 +1871,11 @@ inline void print(std::FILE* f, const S& format_str, Args&&... args) { */ template > inline void print(const S& format_str, Args&&... args) { - return internal::is_unicode() - ? vprint(to_string_view(format_str), - internal::make_args_checked(format_str, args...)) - : internal::vprint_mojibake( - stdout, to_string_view(format_str), - internal::make_args_checked(format_str, args...)); + const auto& vargs = detail::make_args_checked(format_str, args...); + return detail::is_unicode() + ? vprint(to_string_view(format_str), vargs) + : detail::vprint_mojibake(stdout, to_string_view(format_str), + vargs); } FMT_END_NAMESPACE diff --git a/src/fmt/format-inl.h b/src/fmt/format-inl.h index f632714d81..d8c9c8a5ee 100644 --- a/src/fmt/format-inl.h +++ b/src/fmt/format-inl.h @@ -15,6 +15,7 @@ #include #include // for std::memmove #include +#include #include "format.h" #if !defined(FMT_STATIC_THOUSANDS_SEPARATOR) @@ -22,8 +23,16 @@ #endif #ifdef _WIN32 +# if !defined(NOMINMAX) && !defined(WIN32_LEAN_AND_MEAN) +# define NOMINMAX +# define WIN32_LEAN_AND_MEAN +# include +# undef WIN32_LEAN_AND_MEAN +# undef NOMINMAX +# else +# include +# endif # include -# include #endif #ifdef _MSC_VER @@ -33,15 +42,19 @@ // Dummy implementations of strerror_r and strerror_s called if corresponding // system functions are not available. -inline fmt::internal::null<> strerror_r(int, char*, ...) { return {}; } -inline fmt::internal::null<> strerror_s(char*, std::size_t, ...) { return {}; } +inline fmt::detail::null<> strerror_r(int, char*, ...) { return {}; } +inline fmt::detail::null<> strerror_s(char*, size_t, ...) { return {}; } FMT_BEGIN_NAMESPACE -namespace internal { +namespace detail { FMT_FUNC void assert_fail(const char* file, int line, const char* message) { - print(stderr, "{}:{}: assertion failed: {}", file, line, message); - std::abort(); + // Use unchecked std::fprintf to avoid triggering another assertion when + // writing to stderr fails + std::fprintf(stderr, "%s:%d: assertion failed: %s", file, line, message); + // Chosen instead of std::abort to satisfy Clang in CUDA mode during device + // code pass. + std::terminate(); } #ifndef _MSC_VER @@ -67,14 +80,14 @@ inline int fmt_snprintf(char* buffer, size_t size, const char* format, ...) { // other - failure // Buffer should be at least of size 1. FMT_FUNC int safe_strerror(int error_code, char*& buffer, - std::size_t buffer_size) FMT_NOEXCEPT { + size_t buffer_size) FMT_NOEXCEPT { FMT_ASSERT(buffer != nullptr && buffer_size != 0, "invalid buffer"); class dispatcher { private: int error_code_; char*& buffer_; - std::size_t buffer_size_; + size_t buffer_size_; // A noop assignment operator to avoid bogus warnings. void operator=(const dispatcher&) {} @@ -97,7 +110,7 @@ FMT_FUNC int safe_strerror(int error_code, char*& buffer, // Handle the case when strerror_r is not available. FMT_MAYBE_UNUSED - int handle(internal::null<>) { + int handle(detail::null<>) { return fallback(strerror_s(buffer_, buffer_size_, error_code_)); } @@ -111,7 +124,7 @@ FMT_FUNC int safe_strerror(int error_code, char*& buffer, #if !FMT_MSC_VER // Fallback to strerror if strerror_r and strerror_s are not available. - int fallback(internal::null<>) { + int fallback(detail::null<>) { errno = 0; buffer_ = strerror(error_code_); return errno; @@ -119,7 +132,7 @@ FMT_FUNC int safe_strerror(int error_code, char*& buffer, #endif public: - dispatcher(int err_code, char*& buf, std::size_t buf_size) + dispatcher(int err_code, char*& buf, size_t buf_size) : error_code_(err_code), buffer_(buf), buffer_size_(buf_size) {} int run() { return handle(strerror_r(error_code_, buffer_, buffer_size_)); } @@ -127,7 +140,7 @@ FMT_FUNC int safe_strerror(int error_code, char*& buffer, return dispatcher(error_code, buffer, buffer_size).run(); } -FMT_FUNC void format_error_code(internal::buffer& out, int error_code, +FMT_FUNC void format_error_code(detail::buffer& out, int error_code, string_view message) FMT_NOEXCEPT { // Report error code making sure that the output fits into // inline_buffer_size to avoid dynamic memory allocation and potential @@ -136,20 +149,17 @@ FMT_FUNC void format_error_code(internal::buffer& out, int error_code, static const char SEP[] = ": "; static const char ERROR_STR[] = "error "; // Subtract 2 to account for terminating null characters in SEP and ERROR_STR. - std::size_t error_code_size = sizeof(SEP) + sizeof(ERROR_STR) - 2; + size_t error_code_size = sizeof(SEP) + sizeof(ERROR_STR) - 2; auto abs_value = static_cast>(error_code); - if (internal::is_negative(error_code)) { + if (detail::is_negative(error_code)) { abs_value = 0 - abs_value; ++error_code_size; } - error_code_size += internal::to_unsigned(internal::count_digits(abs_value)); - internal::writer w(out); - if (message.size() <= inline_buffer_size - error_code_size) { - w.write(message); - w.write(SEP); - } - w.write(ERROR_STR); - w.write(error_code); + error_code_size += detail::to_unsigned(detail::count_digits(abs_value)); + auto it = std::back_inserter(out); + if (message.size() <= inline_buffer_size - error_code_size) + format_to(it, "{}{}", message, SEP); + format_to(it, "{}{}", ERROR_STR, error_code); assert(out.size() <= inline_buffer_size); } @@ -168,10 +178,10 @@ FMT_FUNC void fwrite_fully(const void* ptr, size_t size, size_t count, size_t written = std::fwrite(ptr, size, count, stream); if (written < count) FMT_THROW(system_error(errno, "cannot write to file")); } -} // namespace internal +} // namespace detail #if !defined(FMT_STATIC_THOUSANDS_SEPARATOR) -namespace internal { +namespace detail { template locale_ref::locale_ref(const Locale& loc) : locale_(&loc) { @@ -194,18 +204,16 @@ template FMT_FUNC Char decimal_point_impl(locale_ref loc) { return std::use_facet>(loc.get()) .decimal_point(); } -} // namespace internal +} // namespace detail #else template -FMT_FUNC std::string internal::grouping_impl(locale_ref) { +FMT_FUNC std::string detail::grouping_impl(locale_ref) { return "\03"; } -template -FMT_FUNC Char internal::thousands_sep_impl(locale_ref) { +template FMT_FUNC Char detail::thousands_sep_impl(locale_ref) { return FMT_STATIC_THOUSANDS_SEPARATOR; } -template -FMT_FUNC Char internal::decimal_point_impl(locale_ref) { +template FMT_FUNC Char detail::decimal_point_impl(locale_ref) { return '.'; } #endif @@ -222,9 +230,9 @@ FMT_FUNC void system_error::init(int err_code, string_view format_str, base = std::runtime_error(to_string(buffer)); } -namespace internal { +namespace detail { -template <> FMT_FUNC int count_digits<4>(internal::fallback_uintptr n) { +template <> FMT_FUNC int count_digits<4>(detail::fallback_uintptr n) { // fallback_uintptr is always stored in little endian. int i = static_cast(sizeof(void*)) - 1; while (i > 0 && n.value[i] == 0) --i; @@ -233,12 +241,27 @@ template <> FMT_FUNC int count_digits<4>(internal::fallback_uintptr n) { } template -const char basic_data::digits[] = - "0001020304050607080910111213141516171819" - "2021222324252627282930313233343536373839" - "4041424344454647484950515253545556575859" - "6061626364656667686970717273747576777879" - "8081828384858687888990919293949596979899"; +const typename basic_data::digit_pair basic_data::digits[] = { + {'0', '0'}, {'0', '1'}, {'0', '2'}, {'0', '3'}, {'0', '4'}, + {'0', '5'}, {'0', '6'}, {'0', '7'}, {'0', '8'}, {'0', '9'}, + {'1', '0'}, {'1', '1'}, {'1', '2'}, {'1', '3'}, {'1', '4'}, + {'1', '5'}, {'1', '6'}, {'1', '7'}, {'1', '8'}, {'1', '9'}, + {'2', '0'}, {'2', '1'}, {'2', '2'}, {'2', '3'}, {'2', '4'}, + {'2', '5'}, {'2', '6'}, {'2', '7'}, {'2', '8'}, {'2', '9'}, + {'3', '0'}, {'3', '1'}, {'3', '2'}, {'3', '3'}, {'3', '4'}, + {'3', '5'}, {'3', '6'}, {'3', '7'}, {'3', '8'}, {'3', '9'}, + {'4', '0'}, {'4', '1'}, {'4', '2'}, {'4', '3'}, {'4', '4'}, + {'4', '5'}, {'4', '6'}, {'4', '7'}, {'4', '8'}, {'4', '9'}, + {'5', '0'}, {'5', '1'}, {'5', '2'}, {'5', '3'}, {'5', '4'}, + {'5', '5'}, {'5', '6'}, {'5', '7'}, {'5', '8'}, {'5', '9'}, + {'6', '0'}, {'6', '1'}, {'6', '2'}, {'6', '3'}, {'6', '4'}, + {'6', '5'}, {'6', '6'}, {'6', '7'}, {'6', '8'}, {'6', '9'}, + {'7', '0'}, {'7', '1'}, {'7', '2'}, {'7', '3'}, {'7', '4'}, + {'7', '5'}, {'7', '6'}, {'7', '7'}, {'7', '8'}, {'7', '9'}, + {'8', '0'}, {'8', '1'}, {'8', '2'}, {'8', '3'}, {'8', '4'}, + {'8', '5'}, {'8', '6'}, {'8', '7'}, {'8', '8'}, {'8', '9'}, + {'9', '0'}, {'9', '1'}, {'9', '2'}, {'9', '3'}, {'9', '4'}, + {'9', '5'}, {'9', '6'}, {'9', '7'}, {'9', '8'}, {'9', '9'}}; template const char basic_data::hex_digits[] = "0123456789abcdef"; @@ -317,6 +340,10 @@ const char basic_data::background_color[] = "\x1b[48;2;"; template const char basic_data::reset_color[] = "\x1b[0m"; template const wchar_t basic_data::wreset_color[] = L"\x1b[0m"; template const char basic_data::signs[] = {0, '-', '+', ' '}; +template +const char basic_data::left_padding_shifts[] = {31, 31, 0, 1, 0}; +template +const char basic_data::right_padding_shifts[] = {0, 31, 0, 1, 0}; template struct bits { static FMT_CONSTEXPR_DECL const int value = @@ -576,9 +603,10 @@ class bigint { void operator=(const bigint&) = delete; void assign(const bigint& other) { - bigits_.resize(other.bigits_.size()); + auto size = other.bigits_.size(); + bigits_.resize(size); auto data = other.bigits_.data(); - std::copy(data, data + other.bigits_.size(), bigits_.data()); + std::copy(data, data + size, make_checked(bigits_.data(), size)); exp_ = other.exp_; } @@ -594,7 +622,7 @@ class bigint { int num_bigits() const { return static_cast(bigits_.size()) + exp_; } - bigint& operator<<=(int shift) { + FMT_NOINLINE bigint& operator<<=(int shift) { assert(shift >= 0); exp_ += shift / bigit_bits; shift %= bigit_bits; @@ -1125,7 +1153,7 @@ int snprintf_float(T value, int precision, float_specs specs, precision = (precision >= 0 ? precision : 6) - 1; // Build the format string. - enum { max_format_size = 7 }; // Ths longest format is "%#.*Le". + enum { max_format_size = 7 }; // The longest format is "%#.*Le". char format[max_format_size]; char* format_ptr = format; *format_ptr++ = '%'; @@ -1145,13 +1173,13 @@ int snprintf_float(T value, int precision, float_specs specs, for (;;) { auto begin = buf.data() + offset; auto capacity = buf.capacity() - offset; -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +#ifdef FMT_FUZZ if (precision > 100000) throw std::runtime_error( "fuzz mode - avoid large allocation inside snprintf"); #endif // Suppress the warning about a nonliteral format string. - // Cannot use auto becase of a bug in MinGW (#1532). + // Cannot use auto because of a bug in MinGW (#1532). int (*snprintf_ptr)(char*, size_t, const char*, ...) = FMT_SNPRINTF; int result = precision >= 0 ? snprintf_ptr(begin, capacity, format, precision, value) @@ -1268,14 +1296,14 @@ FMT_FUNC const char* utf8_decode(const char* buf, uint32_t* c, int* e) { return next; } -} // namespace internal +} // namespace detail -template <> struct formatter { +template <> struct formatter { format_parse_context::iterator parse(format_parse_context& ctx) { return ctx.begin(); } - format_context::iterator format(const internal::bigint& n, + format_context::iterator format(const detail::bigint& n, format_context& ctx) { auto out = ctx.out(); bool first = true; @@ -1289,12 +1317,12 @@ template <> struct formatter { out = format_to(out, "{:08x}", value); } if (n.exp_ > 0) - out = format_to(out, "p{}", n.exp_ * internal::bigint::bigit_bits); + out = format_to(out, "p{}", n.exp_ * detail::bigint::bigit_bits); return out; } }; -FMT_FUNC internal::utf8_to_utf16::utf8_to_utf16(string_view s) { +FMT_FUNC detail::utf8_to_utf16::utf8_to_utf16(string_view s) { auto transcode = [this](const char* p) { auto cp = uint32_t(); auto error = 0; @@ -1325,7 +1353,7 @@ FMT_FUNC internal::utf8_to_utf16::utf8_to_utf16(string_view s) { buffer_.push_back(0); } -FMT_FUNC void format_system_error(internal::buffer& out, int error_code, +FMT_FUNC void format_system_error(detail::buffer& out, int error_code, string_view message) FMT_NOEXCEPT { FMT_TRY { memory_buffer buf; @@ -1333,12 +1361,9 @@ FMT_FUNC void format_system_error(internal::buffer& out, int error_code, for (;;) { char* system_message = &buf[0]; int result = - internal::safe_strerror(error_code, system_message, buf.size()); + detail::safe_strerror(error_code, system_message, buf.size()); if (result == 0) { - internal::writer w(out); - w.write(message); - w.write(": "); - w.write(system_message); + format_to(std::back_inserter(out), "{}: {}", message, system_message); return; } if (result != ERANGE) @@ -1350,7 +1375,7 @@ FMT_FUNC void format_system_error(internal::buffer& out, int error_code, format_error_code(out, error_code, message); } -FMT_FUNC void internal::error_handler::on_error(const char* message) { +FMT_FUNC void detail::error_handler::on_error(const char* message) { FMT_THROW(format_error(message)); } @@ -1359,14 +1384,39 @@ FMT_FUNC void report_system_error(int error_code, report_error(format_system_error, error_code, message); } +struct stringifier { + template FMT_INLINE std::string operator()(T value) const { + return to_string(value); + } + std::string operator()(basic_format_arg::handle h) const { + memory_buffer buf; + detail::buffer& base = buf; + format_parse_context parse_ctx({}); + format_context format_ctx(std::back_inserter(base), {}, {}); + h.format(parse_ctx, format_ctx); + return to_string(buf); + } +}; + +FMT_FUNC std::string detail::vformat(string_view format_str, format_args args) { + if (format_str.size() == 2 && equal2(format_str.data(), "{}")) { + auto arg = args.get(0); + if (!arg) error_handler().on_error("argument not found"); + return visit_format_arg(stringifier(), arg); + } + memory_buffer buffer; + detail::vformat_to(buffer, format_str, args); + return to_string(buffer); +} + FMT_FUNC void vprint(std::FILE* f, string_view format_str, format_args args) { memory_buffer buffer; - internal::vformat_to(buffer, format_str, - basic_format_args>(args)); + detail::vformat_to(buffer, format_str, + basic_format_args>(args)); #ifdef _WIN32 auto fd = _fileno(f); if (_isatty(fd)) { - internal::utf8_to_utf16 u16(string_view(buffer.data(), buffer.size())); + detail::utf8_to_utf16 u16(string_view(buffer.data(), buffer.size())); auto written = DWORD(); if (!WriteConsoleW(reinterpret_cast(_get_osfhandle(fd)), u16.c_str(), static_cast(u16.size()), &written, @@ -1376,16 +1426,16 @@ FMT_FUNC void vprint(std::FILE* f, string_view format_str, format_args args) { return; } #endif - internal::fwrite_fully(buffer.data(), 1, buffer.size(), f); + detail::fwrite_fully(buffer.data(), 1, buffer.size(), f); } #ifdef _WIN32 // Print assuming legacy (non-Unicode) encoding. -FMT_FUNC void internal::vprint_mojibake(std::FILE* f, string_view format_str, - format_args args) { +FMT_FUNC void detail::vprint_mojibake(std::FILE* f, string_view format_str, + format_args args) { memory_buffer buffer; - internal::vformat_to(buffer, format_str, - basic_format_args>(args)); + detail::vformat_to(buffer, format_str, + basic_format_args>(args)); fwrite_fully(buffer.data(), 1, buffer.size(), f); } #endif diff --git a/src/fmt/format.h b/src/fmt/format.h index 645e0630cd..5427042b4e 100644 --- a/src/fmt/format.h +++ b/src/fmt/format.h @@ -43,10 +43,6 @@ #include "core.h" -#ifdef FMT_DEPRECATED_INCLUDE_OS -# include "os.h" -#endif - #ifdef __INTEL_COMPILER # define FMT_ICC_VERSION __INTEL_COMPILER #elif defined(__ICL) @@ -76,7 +72,8 @@ #if __cplusplus == 201103L || __cplusplus == 201402L # if defined(__clang__) && !defined(__INTEL_COMPILER) # define FMT_FALLTHROUGH [[clang::fallthrough]] -# elif FMT_GCC_VERSION >= 700 && !defined(__PGI) && !defined(__INTEL_COMPILER) +# elif FMT_GCC_VERSION >= 700 && !defined(__PGI) && !defined(__INTEL_COMPILER) && \ + (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 520) # define FMT_FALLTHROUGH [[gnu::fallthrough]] # else # define FMT_FALLTHROUGH @@ -88,20 +85,28 @@ # define FMT_FALLTHROUGH #endif +#ifndef FMT_MAYBE_UNUSED +# if FMT_HAS_CPP17_ATTRIBUTE(maybe_unused) +# define FMT_MAYBE_UNUSED [[maybe_unused]] +# else +# define FMT_MAYBE_UNUSED +# endif +#endif + #ifndef FMT_THROW # if FMT_EXCEPTIONS # if FMT_MSC_VER || FMT_NVCC FMT_BEGIN_NAMESPACE -namespace internal { +namespace detail { template inline void do_throw(const Exception& x) { // Silence unreachable code warnings in MSVC and NVCC because these // are nearly impossible to fix in a generic code. volatile bool b = true; if (b) throw x; } -} // namespace internal +} // namespace detail FMT_END_NAMESPACE -# define FMT_THROW(x) internal::do_throw(x) +# define FMT_THROW(x) detail::do_throw(x) # else # define FMT_THROW(x) throw x # endif @@ -123,11 +128,10 @@ FMT_END_NAMESPACE #endif #ifndef FMT_USE_USER_DEFINED_LITERALS -// For Intel and NVIDIA compilers both they and the system gcc/msc support UDLs. -# if (FMT_HAS_FEATURE(cxx_user_literals) || FMT_GCC_VERSION >= 407 || \ - FMT_MSC_VER >= 1900) && \ - (!(FMT_ICC_VERSION || FMT_CUDA_VERSION) || FMT_ICC_VERSION >= 1500 || \ - FMT_CUDA_VERSION >= 700) +// EDG based compilers (Intel, NVIDIA, Elbrus, etc), GCC and MSVC support UDLs. +# if (FMT_HAS_FEATURE(cxx_user_literals) || FMT_GCC_VERSION >= 407 || \ + FMT_MSC_VER >= 1900) && \ + (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= /* UDL feature */ 480) # define FMT_USE_USER_DEFINED_LITERALS 1 # else # define FMT_USE_USER_DEFINED_LITERALS 0 @@ -135,12 +139,11 @@ FMT_END_NAMESPACE #endif #ifndef FMT_USE_UDL_TEMPLATE -// EDG front end based compilers (icc, nvcc) and GCC < 6.4 do not propertly +// EDG frontend based compilers (icc, nvcc, etc) and GCC < 6.4 do not properly // support UDL templates and GCC >= 9 warns about them. -# if FMT_USE_USER_DEFINED_LITERALS && FMT_ICC_VERSION == 0 && \ - FMT_CUDA_VERSION == 0 && \ - ((FMT_GCC_VERSION >= 604 && FMT_GCC_VERSION <= 900 && \ - __cplusplus >= 201402L) || \ +# if FMT_USE_USER_DEFINED_LITERALS && \ + (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 501) && \ + ((FMT_GCC_VERSION >= 604 && __cplusplus >= 201402L) || \ FMT_CLANG_VERSION >= 304) # define FMT_USE_UDL_TEMPLATE 1 # else @@ -176,7 +179,7 @@ FMT_END_NAMESPACE # include // _BitScanReverse, _BitScanReverse64 FMT_BEGIN_NAMESPACE -namespace internal { +namespace detail { // Avoid Clang with Microsoft CodeGen's -Wunknown-pragmas warning. # ifndef __clang__ # pragma intrinsic(_BitScanReverse) @@ -189,10 +192,10 @@ inline uint32_t clz(uint32_t x) { // Static analysis complains about using uninitialized data // "r", but the only way that can happen is if "x" is 0, // which the callers guarantee to not happen. -# pragma warning(suppress : 6102) + FMT_SUPPRESS_MSC_WARNING(6102) return 31 - r; } -# define FMT_BUILTIN_CLZ(n) internal::clz(n) +# define FMT_BUILTIN_CLZ(n) detail::clz(n) # if defined(_WIN64) && !defined(__clang__) # pragma intrinsic(_BitScanReverse64) @@ -214,26 +217,21 @@ inline uint32_t clzll(uint64_t x) { // Static analysis complains about using uninitialized data // "r", but the only way that can happen is if "x" is 0, // which the callers guarantee to not happen. -# pragma warning(suppress : 6102) + FMT_SUPPRESS_MSC_WARNING(6102) return 63 - r; } -# define FMT_BUILTIN_CLZLL(n) internal::clzll(n) -} // namespace internal +# define FMT_BUILTIN_CLZLL(n) detail::clzll(n) +} // namespace detail FMT_END_NAMESPACE #endif // Enable the deprecated numeric alignment. -#ifndef FMT_NUMERIC_ALIGN -# define FMT_NUMERIC_ALIGN 1 -#endif - -// Enable the deprecated percent specifier. -#ifndef FMT_DEPRECATED_PERCENT -# define FMT_DEPRECATED_PERCENT 0 +#ifndef FMT_DEPRECATED_NUMERIC_ALIGN +# define FMT_DEPRECATED_NUMERIC_ALIGN 0 #endif FMT_BEGIN_NAMESPACE -namespace internal { +namespace detail { // An equivalent of `*reinterpret_cast(&source)` that doesn't have // undefined behavior (e.g. due to type aliasing). @@ -285,14 +283,31 @@ template constexpr T max_value() { template constexpr int num_bits() { return std::numeric_limits::digits; } +// std::numeric_limits::digits may return 0 for 128-bit ints. +template <> constexpr int num_bits() { return 128; } +template <> constexpr int num_bits() { return 128; } template <> constexpr int num_bits() { return static_cast(sizeof(void*) * std::numeric_limits::digits); } +FMT_INLINE void assume(bool condition) { + (void)condition; +#if FMT_HAS_BUILTIN(__builtin_assume) + __builtin_assume(condition); +#endif +} + +// A workaround for gcc 4.8 to make void_t work in a SFINAE context. +template struct void_t_impl { using type = void; }; + +template +using void_t = typename detail::void_t_impl::type; + // An approximation of iterator_t for pre-C++20 systems. template using iterator_t = decltype(std::begin(std::declval())); +template using sentinel_t = decltype(std::end(std::declval())); // Detect the iterator category of *any* given type in a SFINAE-friendly way. // Unfortunately, older implementations of std::iterator_traits are not safe @@ -339,25 +354,39 @@ inline typename Container::value_type* get_data(Container& c) { #if defined(_SECURE_SCL) && _SECURE_SCL // Make a checked iterator to avoid MSVC warnings. template using checked_ptr = stdext::checked_array_iterator; -template checked_ptr make_checked(T* p, std::size_t size) { +template checked_ptr make_checked(T* p, size_t size) { return {p, size}; } #else template using checked_ptr = T*; -template inline T* make_checked(T* p, std::size_t) { return p; } +template inline T* make_checked(T* p, size_t) { return p; } #endif template ::value)> -inline checked_ptr reserve( - std::back_insert_iterator& it, std::size_t n) { +#if FMT_CLANG_VERSION +__attribute__((no_sanitize("undefined"))) +#endif +inline checked_ptr +reserve(std::back_insert_iterator it, size_t n) { Container& c = get_container(it); - std::size_t size = c.size(); + size_t size = c.size(); c.resize(size + n); return make_checked(get_data(c) + size, n); } +template inline Iterator& reserve(Iterator& it, size_t) { + return it; +} + +template ::value)> +inline std::back_insert_iterator base_iterator( + std::back_insert_iterator& it, + checked_ptr) { + return it; +} + template -inline Iterator& reserve(Iterator& it, std::size_t) { +inline Iterator base_iterator(Iterator, Iterator it) { return it; } @@ -365,7 +394,7 @@ inline Iterator& reserve(Iterator& it, std::size_t) { // discards them. class counting_iterator { private: - std::size_t count_; + size_t count_; public: using iterator_category = std::output_iterator_tag; @@ -380,7 +409,7 @@ class counting_iterator { counting_iterator() : count_(0) {} - std::size_t count() const { return count_; } + size_t count() const { return count_; } counting_iterator& operator++() { ++count_; @@ -399,10 +428,10 @@ class counting_iterator { template class truncating_iterator_base { protected: OutputIt out_; - std::size_t limit_; - std::size_t count_; + size_t limit_; + size_t count_; - truncating_iterator_base(OutputIt out, std::size_t limit) + truncating_iterator_base(OutputIt out, size_t limit) : out_(out), limit_(limit), count_(0) {} public: @@ -415,7 +444,7 @@ template class truncating_iterator_base { truncating_iterator_base; // Mark iterator as checked. OutputIt base() const { return out_; } - std::size_t count() const { return count_; } + size_t count() const { return count_; } }; // An output iterator that truncates the output and counts the number of objects @@ -433,7 +462,7 @@ class truncating_iterator public: using value_type = typename truncating_iterator_base::value_type; - truncating_iterator(OutputIt out, std::size_t limit) + truncating_iterator(OutputIt out, size_t limit) : truncating_iterator_base(out, limit) {} truncating_iterator& operator++() { @@ -456,7 +485,7 @@ template class truncating_iterator : public truncating_iterator_base { public: - truncating_iterator(OutputIt out, std::size_t limit) + truncating_iterator(OutputIt out, size_t limit) : truncating_iterator_base(out, limit) {} template truncating_iterator& operator=(T val) { @@ -469,22 +498,6 @@ class truncating_iterator truncating_iterator& operator*() { return *this; } }; -// A range with the specified output iterator and value type. -template -class output_range { - private: - OutputIt it_; - - public: - using value_type = T; - using iterator = OutputIt; - struct sentinel {}; - - explicit output_range(OutputIt it) : it_(it) {} - OutputIt begin() const { return it_; } - sentinel end() const { return {}; } // Sentinel is not used yet. -}; - template inline size_t count_code_points(basic_string_view s) { return s.size(); @@ -523,8 +536,6 @@ inline size_t code_point_index(basic_string_view s, size_t n) { return s.size(); } -inline char8_type to_char8_t(char c) { return static_cast(c); } - template using needs_conversion = bool_constant< std::is_same::value_type, @@ -540,7 +551,8 @@ OutputIt copy_str(InputIt begin, InputIt end, OutputIt it) { template ::value)> OutputIt copy_str(InputIt begin, InputIt end, OutputIt it) { - return std::transform(begin, end, it, to_char8_t); + return std::transform(begin, end, it, + [](char c) { return static_cast(c); }); } #ifndef FMT_USE_GRISU @@ -555,43 +567,13 @@ template constexpr bool use_grisu() { template template void buffer::append(const U* begin, const U* end) { - std::size_t new_size = size_ + to_unsigned(end - begin); + size_t new_size = size_ + to_unsigned(end - begin); reserve(new_size); - std::uninitialized_copy(begin, end, make_checked(ptr_, capacity_) + size_); + std::uninitialized_copy(begin, end, + make_checked(ptr_ + size_, capacity_ - size_)); size_ = new_size; } -} // namespace internal - -// A range with an iterator appending to a buffer. -template -class buffer_range : public internal::output_range< - std::back_insert_iterator>, T> { - public: - using iterator = std::back_insert_iterator>; - using internal::output_range::output_range; - buffer_range(internal::buffer& buf) - : internal::output_range(std::back_inserter(buf)) {} -}; - -class FMT_DEPRECATED u8string_view - : public basic_string_view { - public: - u8string_view(const char* s) - : basic_string_view( - reinterpret_cast(s)) {} - u8string_view(const char* s, size_t count) FMT_NOEXCEPT - : basic_string_view( - reinterpret_cast(s), count) {} -}; - -#if FMT_USE_USER_DEFINED_LITERALS -inline namespace literals { -FMT_DEPRECATED inline basic_string_view operator"" _u( - const char* s, std::size_t n) { - return {reinterpret_cast(s), n}; -} -} // namespace literals -#endif +} // namespace detail // The number of characters to store in the basic_memory_buffer object itself // to avoid dynamic memory allocation. @@ -626,27 +608,30 @@ enum { inline_buffer_size = 500 }; The output can be converted to an ``std::string`` with ``to_string(out)``. \endrst */ -template > -class basic_memory_buffer : private Allocator, public internal::buffer { +class basic_memory_buffer : public detail::buffer { private: T store_[SIZE]; + // Don't inherit from Allocator avoid generating type_info for it. + Allocator alloc_; + // Deallocate memory allocated by the buffer. void deallocate() { T* data = this->data(); - if (data != store_) Allocator::deallocate(data, this->capacity()); + if (data != store_) alloc_.deallocate(data, this->capacity()); } protected: - void grow(std::size_t size) FMT_OVERRIDE; + void grow(size_t size) FMT_OVERRIDE; public: using value_type = T; using const_reference = const T&; explicit basic_memory_buffer(const Allocator& alloc = Allocator()) - : Allocator(alloc) { + : alloc_(alloc) { this->set(store_, SIZE); } ~basic_memory_buffer() FMT_OVERRIDE { deallocate(); } @@ -654,14 +639,13 @@ class basic_memory_buffer : private Allocator, public internal::buffer { private: // Move data from other to this buffer. void move(basic_memory_buffer& other) { - Allocator &this_alloc = *this, &other_alloc = other; - this_alloc = std::move(other_alloc); + alloc_ = std::move(other.alloc_); T* data = other.data(); - std::size_t size = other.size(), capacity = other.capacity(); + size_t size = other.size(), capacity = other.capacity(); if (data == other.store_) { this->set(store_, capacity); std::uninitialized_copy(other.store_, other.store_ + size, - internal::make_checked(store_, capacity)); + detail::make_checked(store_, capacity)); } else { this->set(data, capacity); // Set pointer to the inline array so that delete is not called @@ -693,32 +677,37 @@ class basic_memory_buffer : private Allocator, public internal::buffer { } // Returns a copy of the allocator associated with this buffer. - Allocator get_allocator() const { return *this; } + Allocator get_allocator() const { return alloc_; } }; -template -void basic_memory_buffer::grow(std::size_t size) { -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (size > 1000) throw std::runtime_error("fuzz mode - won't grow that much"); +template +void basic_memory_buffer::grow(size_t size) { +#ifdef FMT_FUZZ + if (size > 5000) throw std::runtime_error("fuzz mode - won't grow that much"); #endif - std::size_t old_capacity = this->capacity(); - std::size_t new_capacity = old_capacity + old_capacity / 2; + size_t old_capacity = this->capacity(); + size_t new_capacity = old_capacity + old_capacity / 2; if (size > new_capacity) new_capacity = size; T* old_data = this->data(); - T* new_data = std::allocator_traits::allocate(*this, new_capacity); + T* new_data = + std::allocator_traits::allocate(alloc_, new_capacity); // The following code doesn't throw, so the raw pointer above doesn't leak. std::uninitialized_copy(old_data, old_data + this->size(), - internal::make_checked(new_data, new_capacity)); + detail::make_checked(new_data, new_capacity)); this->set(new_data, new_capacity); // deallocate must not throw according to the standard, but even if it does, // the buffer already uses the new storage and will deallocate it in // destructor. - if (old_data != store_) Allocator::deallocate(old_data, old_capacity); + if (old_data != store_) alloc_.deallocate(old_data, old_capacity); } using memory_buffer = basic_memory_buffer; using wmemory_buffer = basic_memory_buffer; +template +struct is_contiguous> : std::true_type { +}; + /** A formatting error such as invalid format string. */ FMT_CLASS_API class FMT_API format_error : public std::runtime_error { @@ -733,7 +722,7 @@ class FMT_API format_error : public std::runtime_error { ~format_error() FMT_NOEXCEPT FMT_OVERRIDE; }; -namespace internal { +namespace detail { // Returns true if value is negative, false otherwise. // Same as `value < 0` but doesn't produce warnings if T is an unsigned type. @@ -757,8 +746,8 @@ FMT_CONSTEXPR bool is_supported_floating_point(T) { // represent all values of T. template using uint32_or_64_or_128_t = conditional_t< - std::numeric_limits::digits <= 32, uint32_t, - conditional_t::digits <= 64, uint64_t, uint128_t>>; + num_bits() <= 32, uint32_t, + conditional_t() <= 64, uint64_t, uint128_t>>; // Static data is placed in this class template for the header-only config. template struct FMT_EXTERN_TEMPLATE_API basic_data { @@ -767,16 +756,22 @@ template struct FMT_EXTERN_TEMPLATE_API basic_data { static const uint64_t zero_or_powers_of_10_64[]; static const uint64_t pow10_significands[]; static const int16_t pow10_exponents[]; - static const char digits[]; + // GCC generates slightly better code for pairs than chars. + using digit_pair = char[2]; + static const digit_pair digits[]; static const char hex_digits[]; static const char foreground_color[]; static const char background_color[]; static const char reset_color[5]; static const wchar_t wreset_color[5]; static const char signs[]; + static const char left_padding_shifts[5]; + static const char right_padding_shifts[5]; }; +#ifndef FMT_EXPORTED FMT_EXTERN template struct basic_data; +#endif // This is a struct rather than an alias to avoid shadowing warnings in gcc. struct data : basic_data<> {}; @@ -834,7 +829,7 @@ template inline int count_digits(UInt n) { return num_digits; } -template <> int count_digits<4>(internal::fallback_uintptr n); +template <> int count_digits<4>(detail::fallback_uintptr n); #if FMT_GCC_VERSION || FMT_CLANG_VERSION # define FMT_ALWAYS_INLINE inline __attribute__((always_inline)) @@ -850,6 +845,12 @@ inline int count_digits(uint32_t n) { } #endif +template constexpr int digits10() FMT_NOEXCEPT { + return std::numeric_limits::digits10; +} +template <> constexpr int digits10() FMT_NOEXCEPT { return 38; } +template <> constexpr int digits10() FMT_NOEXCEPT { return 38; } + template FMT_API std::string grouping_impl(locale_ref loc); template inline std::string grouping(locale_ref loc) { return grouping_impl(loc); @@ -874,57 +875,61 @@ template <> inline wchar_t decimal_point(locale_ref loc) { return decimal_point_impl(loc); } -// Formats a decimal unsigned integer value writing into buffer. -// add_thousands_sep is called after writing each char to add a thousands -// separator if necessary. -template -inline Char* format_decimal(Char* buffer, UInt value, int num_digits, - F add_thousands_sep) { - FMT_ASSERT(num_digits >= 0, "invalid digit count"); - buffer += num_digits; - Char* end = buffer; +// Compares two characters for equality. +template bool equal2(const Char* lhs, const char* rhs) { + return lhs[0] == rhs[0] && lhs[1] == rhs[1]; +} +inline bool equal2(const char* lhs, const char* rhs) { + return memcmp(lhs, rhs, 2) == 0; +} + +// Copies two characters from src to dst. +template void copy2(Char* dst, const char* src) { + *dst++ = static_cast(*src++); + *dst = static_cast(*src); +} +inline void copy2(char* dst, const char* src) { memcpy(dst, src, 2); } + +template struct format_decimal_result { + Iterator begin; + Iterator end; +}; + +// Formats a decimal unsigned integer value writing into out pointing to a +// buffer of specified size. The caller must ensure that the buffer is large +// enough. +template +inline format_decimal_result format_decimal(Char* out, UInt value, + int size) { + FMT_ASSERT(size >= count_digits(value), "invalid digit count"); + out += size; + Char* end = out; while (value >= 100) { // Integer division is slow so do it for a group of two digits instead // of for every digit. The idea comes from the talk by Alexandrescu // "Three Optimization Tips for C++". See speed-test for a comparison. - auto index = static_cast((value % 100) * 2); + out -= 2; + copy2(out, data::digits[value % 100]); value /= 100; - *--buffer = static_cast(data::digits[index + 1]); - add_thousands_sep(buffer); - *--buffer = static_cast(data::digits[index]); - add_thousands_sep(buffer); } if (value < 10) { - *--buffer = static_cast('0' + value); - return end; + *--out = static_cast('0' + value); + return {out, end}; } - auto index = static_cast(value * 2); - *--buffer = static_cast(data::digits[index + 1]); - add_thousands_sep(buffer); - *--buffer = static_cast(data::digits[index]); - return end; + out -= 2; + copy2(out, data::digits[value]); + return {out, end}; } -template constexpr int digits10() FMT_NOEXCEPT { - return std::numeric_limits::digits10; -} -template <> constexpr int digits10() FMT_NOEXCEPT { return 38; } -template <> constexpr int digits10() FMT_NOEXCEPT { return 38; } - -template -inline Iterator format_decimal(Iterator out, UInt value, int num_digits, - F add_thousands_sep) { - FMT_ASSERT(num_digits >= 0, "invalid digit count"); +template >::value)> +inline format_decimal_result format_decimal(Iterator out, UInt value, + int num_digits) { // Buffer should be large enough to hold all digits (<= digits10 + 1). enum { max_size = digits10() + 1 }; Char buffer[2 * max_size]; - auto end = format_decimal(buffer, value, num_digits, add_thousands_sep); - return internal::copy_str(buffer, end, out); -} - -template -inline It format_decimal(It out, UInt value, int num_digits) { - return format_decimal(out, value, num_digits, [](Char*) {}); + auto end = format_decimal(buffer, value, num_digits).end; + return {out, detail::copy_str(buffer, end, out)}; } template @@ -942,7 +947,7 @@ inline Char* format_uint(Char* buffer, UInt value, int num_digits, } template -Char* format_uint(Char* buffer, internal::fallback_uintptr n, int num_digits, +Char* format_uint(Char* buffer, detail::fallback_uintptr n, int num_digits, bool = false) { auto char_digits = std::numeric_limits::digits / 4; int start = (num_digits + char_digits - 1) / char_digits - 1; @@ -968,7 +973,7 @@ inline It format_uint(It out, UInt value, int num_digits, bool upper = false) { // Buffer should be large enough to hold all digits (digits / BASE_BITS + 1). char buffer[num_bits() / BASE_BITS + 1]; format_uint(buffer, value, num_digits, upper); - return internal::copy_str(buffer, buffer + num_digits, out); + return detail::copy_str(buffer, buffer + num_digits, out); } // A converter from UTF-8 to UTF-16. @@ -1019,7 +1024,7 @@ template struct fill_t { return fill; } }; -} // namespace internal +} // namespace detail // We cannot use enum classes as bit fields because of a gcc bug // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414. @@ -1041,7 +1046,7 @@ template struct basic_format_specs { align_t align : 4; sign_t sign : 3; bool alt : 1; // Alternate form ('#'). - internal::fill_t fill; + detail::fill_t fill; constexpr basic_format_specs() : width(0), @@ -1050,12 +1055,12 @@ template struct basic_format_specs { align(align::none), sign(sign::none), alt(false), - fill(internal::fill_t::make()) {} + fill(detail::fill_t::make()) {} }; using format_specs = basic_format_specs; -namespace internal { +namespace detail { // A floating-point presentation format. enum class float_format : unsigned char { @@ -1071,7 +1076,6 @@ struct float_specs { sign_t sign : 8; bool upper : 1; bool locale : 1; - bool percent : 1; bool binary32 : 1; bool use_grisu : 1; bool showpoint : 1; @@ -1087,12 +1091,12 @@ template It write_exponent(int exp, It it) { *it++ = static_cast('+'); } if (exp >= 100) { - const char* top = data::digits + (exp / 100) * 2; + const char* top = data::digits[exp / 100]; if (exp >= 1000) *it++ = static_cast(top[0]); *it++ = static_cast(top[1]); exp %= 100; } - const char* d = data::digits + exp * 2; + const char* d = data::digits[exp]; *it++ = static_cast(d[0]); *it++ = static_cast(d[1]); return it; @@ -1134,8 +1138,8 @@ template class float_writer { *it++ = static_cast('0'); return it; } -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (num_zeros > 1000) +#ifdef FMT_FUZZ + if (num_zeros > 5000) throw std::runtime_error("fuzz mode - avoiding excessive cpu use"); #endif it = std::fill_n(it, num_zeros, static_cast('0')); @@ -1198,11 +1202,10 @@ template class float_writer { } size_t size() const { return size_; } - size_t width() const { return size(); } - template void operator()(It&& it) { + template It operator()(It it) const { if (specs_.sign) *it++ = static_cast(data::signs[specs_.sign]); - it = prettify(it); + return prettify(it); } }; @@ -1235,10 +1238,15 @@ FMT_CONSTEXPR void handle_int_type_spec(char spec, Handler&& handler) { case 'o': handler.on_oct(); break; +#ifdef FMT_DEPRECATED_N_SPECIFIER case 'n': +#endif case 'L': handler.on_num(); break; + case 'c': + handler.on_chr(); + break; default: handler.on_error(); } @@ -1274,19 +1282,16 @@ FMT_CONSTEXPR float_specs parse_float_type_spec( result.format = float_format::fixed; result.showpoint |= specs.precision != 0; break; -#if FMT_DEPRECATED_PERCENT - case '%': - result.format = float_format::fixed; - result.percent = true; - break; -#endif case 'A': result.upper = true; FMT_FALLTHROUGH; case 'a': result.format = float_format::hex; break; +#ifdef FMT_DEPRECATED_N_SPECIFIER case 'n': +#endif + case 'L': result.locale = true; break; default: @@ -1335,6 +1340,7 @@ template class int_type_checker : private ErrorHandler { FMT_CONSTEXPR void on_bin() {} FMT_CONSTEXPR void on_oct() {} FMT_CONSTEXPR void on_num() {} + FMT_CONSTEXPR void on_chr() {} FMT_CONSTEXPR void on_error() { ErrorHandler::on_error("invalid type specifier"); @@ -1366,38 +1372,6 @@ class cstring_type_checker : public ErrorHandler { FMT_CONSTEXPR void on_pointer() {} }; -template -void arg_map::init(const basic_format_args& args) { - if (map_) return; - map_ = new entry[internal::to_unsigned(args.max_size())]; - if (args.is_packed()) { - for (int i = 0;; ++i) { - internal::type arg_type = args.type(i); - if (arg_type == internal::type::none_type) return; - if (arg_type == internal::type::named_arg_type) - push_back(args.values_[i]); - } - } - for (int i = 0, n = args.max_size(); i < n; ++i) { - auto type = args.args_[i].type_; - if (type == internal::type::named_arg_type) push_back(args.args_[i].value_); - } -} - -template struct nonfinite_writer { - sign_t sign; - const char* str; - static constexpr size_t str_size = 3; - - size_t size() const { return str_size + (sign ? 1 : 0); } - size_t width() const { return size(); } - - template void operator()(It&& it) const { - if (sign) *it++ = static_cast(data::signs[sign]); - it = copy_str(str, str + str_size, it); - } -}; - template FMT_NOINLINE OutputIt fill(OutputIt it, size_t n, const fill_t& fill) { auto fill_size = fill.size(); @@ -1406,375 +1380,470 @@ FMT_NOINLINE OutputIt fill(OutputIt it, size_t n, const fill_t& fill) { return it; } -// This template provides operations for formatting and writing data into a -// character range. -template class basic_writer { - public: - using char_type = typename Range::value_type; - using iterator = typename Range::iterator; - using format_specs = basic_format_specs; +// Writes the output of f, padded according to format specifications in specs. +// size: output size in code units. +// width: output display width in (terminal) column positions. +template +inline OutputIt write_padded(OutputIt out, + const basic_format_specs& specs, size_t size, + size_t width, const F& f) { + static_assert(align == align::left || align == align::right, ""); + unsigned spec_width = to_unsigned(specs.width); + size_t padding = spec_width > width ? spec_width - width : 0; + auto* shifts = align == align::left ? data::left_padding_shifts + : data::right_padding_shifts; + size_t left_padding = padding >> shifts[specs.align]; + auto it = reserve(out, size + padding * specs.fill.size()); + it = fill(it, left_padding, specs.fill); + it = f(it); + it = fill(it, padding - left_padding, specs.fill); + return base_iterator(out, it); +} - private: - iterator out_; // Output iterator. - locale_ref locale_; +template +inline OutputIt write_padded(OutputIt out, + const basic_format_specs& specs, size_t size, + const F& f) { + return write_padded(out, specs, size, size, f); +} - // Attempts to reserve space for n extra characters in the output range. - // Returns a pointer to the reserved range or a reference to out_. - auto reserve(std::size_t n) -> decltype(internal::reserve(out_, n)) { - return internal::reserve(out_, n); - } +template +OutputIt write_bytes(OutputIt out, string_view bytes, + const basic_format_specs& specs) { + using iterator = remove_reference_t; + return write_padded(out, specs, bytes.size(), [bytes](iterator it) { + const char* data = bytes.data(); + return copy_str(data, data + bytes.size(), it); + }); +} - template struct padded_int_writer { - size_t size_; - string_view prefix; - char_type fill; - std::size_t padding; - F f; +// Data for write_int that doesn't depend on output iterator type. It is used to +// avoid template code bloat. +template struct write_int_data { + size_t size; + size_t padding; - size_t size() const { return size_; } - size_t width() const { return size_; } - - template void operator()(It&& it) const { - if (prefix.size() != 0) - it = copy_str(prefix.begin(), prefix.end(), it); - it = std::fill_n(it, padding, fill); - f(it); - } - }; - - // Writes an integer in the format - // - // where are written by f(it). - template - void write_int(int num_digits, string_view prefix, format_specs specs, F f) { - std::size_t size = prefix.size() + to_unsigned(num_digits); - char_type fill = specs.fill[0]; - std::size_t padding = 0; + write_int_data(int num_digits, string_view prefix, + const basic_format_specs& specs) + : size(prefix.size() + to_unsigned(num_digits)), padding(0) { if (specs.align == align::numeric) { - auto unsiged_width = to_unsigned(specs.width); - if (unsiged_width > size) { - padding = unsiged_width - size; - size = unsiged_width; + auto width = to_unsigned(specs.width); + if (width > size) { + padding = width - size; + size = width; } } else if (specs.precision > num_digits) { size = prefix.size() + to_unsigned(specs.precision); padding = to_unsigned(specs.precision - num_digits); - fill = static_cast('0'); } - if (specs.align == align::none) specs.align = align::right; - write_padded(specs, padded_int_writer{size, prefix, fill, padding, f}); + } +}; + +// Writes an integer in the format +// +// where are written by f(it). +template +OutputIt write_int(OutputIt out, int num_digits, string_view prefix, + const basic_format_specs& specs, F f) { + auto data = write_int_data(num_digits, prefix, specs); + using iterator = remove_reference_t; + return write_padded(out, specs, data.size, [=](iterator it) { + if (prefix.size() != 0) + it = copy_str(prefix.begin(), prefix.end(), it); + it = std::fill_n(it, data.padding, static_cast('0')); + return f(it); + }); +} + +template +OutputIt write(OutputIt out, basic_string_view s, + const basic_format_specs& specs) { + auto data = s.data(); + auto size = s.size(); + if (specs.precision >= 0 && to_unsigned(specs.precision) < size) + size = code_point_index(s, to_unsigned(specs.precision)); + auto width = specs.width != 0 + ? count_code_points(basic_string_view(data, size)) + : 0; + using iterator = remove_reference_t; + return write_padded(out, specs, size, width, [=](iterator it) { + return copy_str(data, data + size, it); + }); +} + +// The handle_int_type_spec handler that writes an integer. +template struct int_writer { + OutputIt out; + locale_ref locale; + const basic_format_specs& specs; + UInt abs_value; + char prefix[4]; + unsigned prefix_size; + + using iterator = + remove_reference_t(), 0))>; + + string_view get_prefix() const { return string_view(prefix, prefix_size); } + + template + int_writer(OutputIt output, locale_ref loc, Int value, + const basic_format_specs& s) + : out(output), + locale(loc), + specs(s), + abs_value(static_cast(value)), + prefix_size(0) { + static_assert(std::is_same, UInt>::value, ""); + if (is_negative(value)) { + prefix[0] = '-'; + ++prefix_size; + abs_value = 0 - abs_value; + } else if (specs.sign != sign::none && specs.sign != sign::minus) { + prefix[0] = specs.sign == sign::plus ? '+' : ' '; + ++prefix_size; + } } - // Writes a decimal integer. - template void write_decimal(Int value) { - auto abs_value = static_cast>(value); - bool negative = is_negative(value); - // Don't do -abs_value since it trips unsigned-integer-overflow sanitizer. - if (negative) abs_value = ~abs_value + 1; + void on_dec() { + auto num_digits = count_digits(abs_value); + out = write_int( + out, num_digits, get_prefix(), specs, [this, num_digits](iterator it) { + return format_decimal(it, abs_value, num_digits).end; + }); + } + + void on_hex() { + if (specs.alt) { + prefix[prefix_size++] = '0'; + prefix[prefix_size++] = specs.type; + } + int num_digits = count_digits<4>(abs_value); + out = write_int(out, num_digits, get_prefix(), specs, + [this, num_digits](iterator it) { + return format_uint<4, Char>(it, abs_value, num_digits, + specs.type != 'x'); + }); + } + + void on_bin() { + if (specs.alt) { + prefix[prefix_size++] = '0'; + prefix[prefix_size++] = static_cast(specs.type); + } + int num_digits = count_digits<1>(abs_value); + out = write_int(out, num_digits, get_prefix(), specs, + [this, num_digits](iterator it) { + return format_uint<1, Char>(it, abs_value, num_digits); + }); + } + + void on_oct() { + int num_digits = count_digits<3>(abs_value); + if (specs.alt && specs.precision <= num_digits && abs_value != 0) { + // Octal prefix '0' is counted as a digit, so only add it if precision + // is not greater than the number of digits. + prefix[prefix_size++] = '0'; + } + out = write_int(out, num_digits, get_prefix(), specs, + [this, num_digits](iterator it) { + return format_uint<3, Char>(it, abs_value, num_digits); + }); + } + + enum { sep_size = 1 }; + + void on_num() { + std::string groups = grouping(locale); + if (groups.empty()) return on_dec(); + auto sep = thousands_sep(locale); + if (!sep) return on_dec(); int num_digits = count_digits(abs_value); - auto&& it = reserve((negative ? 1 : 0) + static_cast(num_digits)); - if (negative) *it++ = static_cast('-'); - it = format_decimal(it, abs_value, num_digits); - } - - // The handle_int_type_spec handler that writes an integer. - template struct int_writer { - using unsigned_type = uint32_or_64_or_128_t; - - basic_writer& writer; - const Specs& specs; - unsigned_type abs_value; - char prefix[4]; - unsigned prefix_size; - - string_view get_prefix() const { return string_view(prefix, prefix_size); } - - int_writer(basic_writer& w, Int value, const Specs& s) - : writer(w), - specs(s), - abs_value(static_cast(value)), - prefix_size(0) { - if (is_negative(value)) { - prefix[0] = '-'; - ++prefix_size; - abs_value = 0 - abs_value; - } else if (specs.sign != sign::none && specs.sign != sign::minus) { - prefix[0] = specs.sign == sign::plus ? '+' : ' '; - ++prefix_size; - } + int size = num_digits, n = num_digits; + std::string::const_iterator group = groups.cbegin(); + while (group != groups.cend() && n > *group && *group > 0 && + *group != max_value()) { + size += sep_size; + n -= *group; + ++group; } - - struct dec_writer { - unsigned_type abs_value; - int num_digits; - - template void operator()(It&& it) const { - it = internal::format_decimal(it, abs_value, num_digits); - } - }; - - void on_dec() { - int num_digits = count_digits(abs_value); - writer.write_int(num_digits, get_prefix(), specs, - dec_writer{abs_value, num_digits}); - } - - struct hex_writer { - int_writer& self; - int num_digits; - - template void operator()(It&& it) const { - it = format_uint<4, char_type>(it, self.abs_value, num_digits, - self.specs.type != 'x'); - } - }; - - void on_hex() { - if (specs.alt) { - prefix[prefix_size++] = '0'; - prefix[prefix_size++] = specs.type; - } - int num_digits = count_digits<4>(abs_value); - writer.write_int(num_digits, get_prefix(), specs, - hex_writer{*this, num_digits}); - } - - template struct bin_writer { - unsigned_type abs_value; - int num_digits; - - template void operator()(It&& it) const { - it = format_uint(it, abs_value, num_digits); - } - }; - - void on_bin() { - if (specs.alt) { - prefix[prefix_size++] = '0'; - prefix[prefix_size++] = static_cast(specs.type); - } - int num_digits = count_digits<1>(abs_value); - writer.write_int(num_digits, get_prefix(), specs, - bin_writer<1>{abs_value, num_digits}); - } - - void on_oct() { - int num_digits = count_digits<3>(abs_value); - if (specs.alt && specs.precision <= num_digits && abs_value != 0) { - // Octal prefix '0' is counted as a digit, so only add it if precision - // is not greater than the number of digits. - prefix[prefix_size++] = '0'; - } - writer.write_int(num_digits, get_prefix(), specs, - bin_writer<3>{abs_value, num_digits}); - } - - enum { sep_size = 1 }; - - struct num_writer { - unsigned_type abs_value; - int size; - const std::string& groups; - char_type sep; - - template void operator()(It&& it) const { - basic_string_view s(&sep, sep_size); - // Index of a decimal digit with the least significant digit having - // index 0. - int digit_index = 0; - std::string::const_iterator group = groups.cbegin(); - it = format_decimal( - it, abs_value, size, - [this, s, &group, &digit_index](char_type*& buffer) { - if (*group <= 0 || ++digit_index % *group != 0 || - *group == max_value()) - return; - if (group + 1 != groups.cend()) { - digit_index = 0; - ++group; - } - buffer -= s.size(); - std::uninitialized_copy(s.data(), s.data() + s.size(), - make_checked(buffer, s.size())); - }); - } - }; - - void on_num() { - std::string groups = grouping(writer.locale_); - if (groups.empty()) return on_dec(); - auto sep = thousands_sep(writer.locale_); - if (!sep) return on_dec(); - int num_digits = count_digits(abs_value); - int size = num_digits; - std::string::const_iterator group = groups.cbegin(); - while (group != groups.cend() && num_digits > *group && *group > 0 && - *group != max_value()) { - size += sep_size; - num_digits -= *group; + if (group == groups.cend()) size += sep_size * ((n - 1) / groups.back()); + char digits[40]; + format_decimal(digits, abs_value, num_digits); + basic_memory_buffer buffer; + size += prefix_size; + buffer.resize(size); + basic_string_view s(&sep, sep_size); + // Index of a decimal digit with the least significant digit having index 0. + int digit_index = 0; + group = groups.cbegin(); + auto p = buffer.data() + size; + for (int i = num_digits - 1; i >= 0; --i) { + *--p = static_cast(digits[i]); + if (*group <= 0 || ++digit_index % *group != 0 || + *group == max_value()) + continue; + if (group + 1 != groups.cend()) { + digit_index = 0; ++group; } - if (group == groups.cend()) - size += sep_size * ((num_digits - 1) / groups.back()); - writer.write_int(size, get_prefix(), specs, - num_writer{abs_value, size, groups, sep}); + p -= s.size(); + std::uninitialized_copy(s.data(), s.data() + s.size(), + make_checked(p, s.size())); } + if (prefix_size != 0) p[-1] = static_cast('-'); + write(out, basic_string_view(buffer.data(), buffer.size()), specs); + } - FMT_NORETURN void on_error() { - FMT_THROW(format_error("invalid type specifier")); - } + void on_chr() { *out++ = static_cast(abs_value); } + + FMT_NORETURN void on_error() { + FMT_THROW(format_error("invalid type specifier")); + } +}; + +template +OutputIt write_nonfinite(OutputIt out, bool isinf, + const basic_format_specs& specs, + const float_specs& fspecs) { + auto str = + isinf ? (fspecs.upper ? "INF" : "inf") : (fspecs.upper ? "NAN" : "nan"); + constexpr size_t str_size = 3; + auto sign = fspecs.sign; + auto size = str_size + (sign ? 1 : 0); + using iterator = remove_reference_t; + return write_padded(out, specs, size, [=](iterator it) { + if (sign) *it++ = static_cast(data::signs[sign]); + return copy_str(str, str + str_size, it); + }); +} + +template ::value)> +OutputIt write(OutputIt out, T value, basic_format_specs specs, + locale_ref loc = {}) { + if (const_check(!is_supported_floating_point(value))) return out; + float_specs fspecs = parse_float_type_spec(specs); + fspecs.sign = specs.sign; + if (std::signbit(value)) { // value < 0 is false for NaN so use signbit. + fspecs.sign = sign::minus; + value = -value; + } else if (fspecs.sign == sign::minus) { + fspecs.sign = sign::none; + } + + if (!std::isfinite(value)) + return write_nonfinite(out, std::isinf(value), specs, fspecs); + + if (specs.align == align::numeric && fspecs.sign) { + auto it = reserve(out, 1); + *it++ = static_cast(data::signs[fspecs.sign]); + out = base_iterator(out, it); + fspecs.sign = sign::none; + if (specs.width != 0) --specs.width; + } + + memory_buffer buffer; + if (fspecs.format == float_format::hex) { + if (fspecs.sign) buffer.push_back(data::signs[fspecs.sign]); + snprintf_float(promote_float(value), specs.precision, fspecs, buffer); + return write_bytes(out, {buffer.data(), buffer.size()}, specs); + } + int precision = specs.precision >= 0 || !specs.type ? specs.precision : 6; + if (fspecs.format == float_format::exp) { + if (precision == max_value()) + FMT_THROW(format_error("number is too big")); + else + ++precision; + } + if (const_check(std::is_same())) fspecs.binary32 = true; + fspecs.use_grisu = use_grisu(); + int exp = format_float(promote_float(value), precision, fspecs, buffer); + fspecs.precision = precision; + Char point = + fspecs.locale ? decimal_point(loc) : static_cast('.'); + float_writer w(buffer.data(), static_cast(buffer.size()), exp, + fspecs, point); + return write_padded(out, specs, w.size(), w); +} + +template ::value)> +OutputIt write(OutputIt out, T value) { + if (const_check(!is_supported_floating_point(value))) return out; + auto fspecs = float_specs(); + if (std::signbit(value)) { // value < 0 is false for NaN so use signbit. + fspecs.sign = sign::minus; + value = -value; + } + + auto specs = basic_format_specs(); + if (!std::isfinite(value)) + return write_nonfinite(out, std::isinf(value), specs, fspecs); + + memory_buffer buffer; + int precision = -1; + if (const_check(std::is_same())) fspecs.binary32 = true; + fspecs.use_grisu = use_grisu(); + int exp = format_float(promote_float(value), precision, fspecs, buffer); + fspecs.precision = precision; + float_writer w(buffer.data(), static_cast(buffer.size()), exp, + fspecs, static_cast('.')); + return base_iterator(out, w(reserve(out, w.size()))); +} + +template +OutputIt write_char(OutputIt out, Char value, + const basic_format_specs& specs) { + using iterator = remove_reference_t; + return write_padded(out, specs, 1, [=](iterator it) { + *it++ = value; + return it; + }); +} + +template +OutputIt write_ptr(OutputIt out, UIntPtr value, + const basic_format_specs* specs) { + int num_digits = count_digits<4>(value); + auto size = to_unsigned(num_digits) + size_t(2); + using iterator = remove_reference_t; + auto write = [=](iterator it) { + *it++ = static_cast('0'); + *it++ = static_cast('x'); + return format_uint<4, Char>(it, value, num_digits); }; + return specs ? write_padded(out, *specs, size, write) + : base_iterator(out, write(reserve(out, size))); +} - template struct str_writer { - const Char* s; - size_t size_; +template struct is_integral : std::is_integral {}; +template <> struct is_integral : std::true_type {}; +template <> struct is_integral : std::true_type {}; - size_t size() const { return size_; } - size_t width() const { - return count_code_points(basic_string_view(s, size_)); - } +template +OutputIt write(OutputIt out, monostate) { + FMT_ASSERT(false, ""); + return out; +} - template void operator()(It&& it) const { - it = copy_str(s, s + size_, it); - } - }; +template ::value)> +OutputIt write(OutputIt out, string_view value) { + auto it = reserve(out, value.size()); + it = copy_str(value.begin(), value.end(), it); + return base_iterator(out, it); +} - struct bytes_writer { - string_view bytes; +template +OutputIt write(OutputIt out, basic_string_view value) { + auto it = reserve(out, value.size()); + it = std::copy(value.begin(), value.end(), it); + return base_iterator(out, it); +} - size_t size() const { return bytes.size(); } - size_t width() const { return bytes.size(); } +template ::value && + !std::is_same::value && + !std::is_same::value)> +OutputIt write(OutputIt out, T value) { + auto abs_value = static_cast>(value); + bool negative = is_negative(value); + // Don't do -abs_value since it trips unsigned-integer-overflow sanitizer. + if (negative) abs_value = ~abs_value + 1; + int num_digits = count_digits(abs_value); + auto it = reserve(out, (negative ? 1 : 0) + static_cast(num_digits)); + if (negative) *it++ = static_cast('-'); + it = format_decimal(it, abs_value, num_digits).end; + return base_iterator(out, it); +} - template void operator()(It&& it) const { - const char* data = bytes.data(); - it = copy_str(data, data + size(), it); - } - }; +template +OutputIt write(OutputIt out, bool value) { + return write(out, string_view(value ? "true" : "false")); +} - template struct pointer_writer { - UIntPtr value; - int num_digits; +template +OutputIt write(OutputIt out, Char value) { + auto it = reserve(out, 1); + *it++ = value; + return base_iterator(out, it); +} - size_t size() const { return to_unsigned(num_digits) + 2; } - size_t width() const { return size(); } +template +OutputIt write(OutputIt out, const Char* value) { + if (!value) { + FMT_THROW(format_error("string pointer is null")); + } else { + auto length = std::char_traits::length(value); + out = write(out, basic_string_view(value, length)); + } + return out; +} - template void operator()(It&& it) const { - *it++ = static_cast('0'); - *it++ = static_cast('x'); - it = format_uint<4, char_type>(it, value, num_digits); - } - }; +template +OutputIt write(OutputIt out, const void* value) { + return write_ptr(out, to_uintptr(value), nullptr); +} +template +auto write(OutputIt out, const T& value) -> typename std::enable_if< + mapped_type_constant>::value == + type::custom_type, + OutputIt>::type { + basic_format_context ctx(out, {}, {}); + return formatter().format(value, ctx); +} + +// An argument visitor that formats the argument and writes it via the output +// iterator. It's a class and not a generic lambda for compatibility with C++11. +template struct default_arg_formatter { + using context = basic_format_context; + + OutputIt out; + basic_format_args args; + locale_ref loc; + + template OutputIt operator()(T value) { + return write(out, value); + } + + OutputIt operator()(typename basic_format_arg::handle handle) { + basic_format_parse_context parse_ctx({}); + basic_format_context format_ctx(out, args, loc); + handle.format(parse_ctx, format_ctx); + return format_ctx.out(); + } +}; + +template +class arg_formatter_base { public: - explicit basic_writer(Range out, locale_ref loc = locale_ref()) - : out_(out.begin()), locale_(loc) {} + using iterator = OutputIt; + using char_type = Char; + using format_specs = basic_format_specs; - iterator out() const { return out_; } + private: + iterator out_; + locale_ref locale_; + format_specs* specs_; - // Writes a value in the format - // - // where is written by f(it). - template void write_padded(const format_specs& specs, F&& f) { - // User-perceived width (in code points). - unsigned width = to_unsigned(specs.width); - size_t size = f.size(); // The number of code units. - size_t num_code_points = width != 0 ? f.width() : size; - if (width <= num_code_points) return f(reserve(size)); - size_t padding = width - num_code_points; - size_t fill_size = specs.fill.size(); - auto&& it = reserve(size + padding * fill_size); - if (specs.align == align::right) { - it = fill(it, padding, specs.fill); - f(it); - } else if (specs.align == align::center) { - std::size_t left_padding = padding / 2; - it = fill(it, left_padding, specs.fill); - f(it); - it = fill(it, padding - left_padding, specs.fill); - } else { - f(it); - it = fill(it, padding, specs.fill); - } + // Attempts to reserve space for n extra characters in the output range. + // Returns a pointer to the reserved range or a reference to out_. + auto reserve(size_t n) -> decltype(detail::reserve(out_, n)) { + return detail::reserve(out_, n); } - void write(int value) { write_decimal(value); } - void write(long value) { write_decimal(value); } - void write(long long value) { write_decimal(value); } + using reserve_iterator = remove_reference_t(), 0))>; - void write(unsigned value) { write_decimal(value); } - void write(unsigned long value) { write_decimal(value); } - void write(unsigned long long value) { write_decimal(value); } - -#if FMT_USE_INT128 - void write(int128_t value) { write_decimal(value); } - void write(uint128_t value) { write_decimal(value); } -#endif - - template - void write_int(T value, const Spec& spec) { - handle_int_type_spec(spec.type, int_writer(*this, value, spec)); - } - - template ::value)> - void write(T value, format_specs specs = {}) { - if (const_check(!is_supported_floating_point(value))) { - return; - } - float_specs fspecs = parse_float_type_spec(specs); - fspecs.sign = specs.sign; - if (std::signbit(value)) { // value < 0 is false for NaN so use signbit. - fspecs.sign = sign::minus; - value = -value; - } else if (fspecs.sign == sign::minus) { - fspecs.sign = sign::none; - } - - if (!std::isfinite(value)) { - auto str = std::isinf(value) ? (fspecs.upper ? "INF" : "inf") - : (fspecs.upper ? "NAN" : "nan"); - return write_padded(specs, nonfinite_writer{fspecs.sign, str}); - } - - if (specs.align == align::none) { - specs.align = align::right; - } else if (specs.align == align::numeric) { - if (fspecs.sign) { - auto&& it = reserve(1); - *it++ = static_cast(data::signs[fspecs.sign]); - fspecs.sign = sign::none; - if (specs.width != 0) --specs.width; - } - specs.align = align::right; - } - - memory_buffer buffer; - if (fspecs.format == float_format::hex) { - if (fspecs.sign) buffer.push_back(data::signs[fspecs.sign]); - snprintf_float(promote_float(value), specs.precision, fspecs, buffer); - write_padded(specs, str_writer{buffer.data(), buffer.size()}); - return; - } - int precision = specs.precision >= 0 || !specs.type ? specs.precision : 6; - if (fspecs.format == float_format::exp) { - if (precision == max_value()) - FMT_THROW(format_error("number is too big")); - else - ++precision; - } - if (const_check(std::is_same())) fspecs.binary32 = true; - fspecs.use_grisu = use_grisu(); - if (const_check(FMT_DEPRECATED_PERCENT) && fspecs.percent) value *= 100; - int exp = format_float(promote_float(value), precision, fspecs, buffer); - if (const_check(FMT_DEPRECATED_PERCENT) && fspecs.percent) { - buffer.push_back('%'); - --exp; // Adjust decimal place position. - } - fspecs.precision = precision; - char_type point = fspecs.locale ? decimal_point(locale_) - : static_cast('.'); - write_padded(specs, float_writer(buffer.data(), - static_cast(buffer.size()), - exp, fspecs, point)); + template void write_int(T value, const format_specs& spec) { + using uint_type = uint32_or_64_or_128_t; + int_writer w(out_, locale_, value, spec); + handle_int_type_spec(spec.type, w); + out_ = w.out; } void write(char value) { @@ -1782,198 +1851,151 @@ template class basic_writer { *it++ = value; } - template ::value)> - void write(Char value) { - auto&& it = reserve(1); - *it++ = value; + template ::value)> + void write(Ch value) { + out_ = detail::write(out_, value); } void write(string_view value) { auto&& it = reserve(value.size()); - it = copy_str(value.begin(), value.end(), it); + it = copy_str(value.begin(), value.end(), it); } void write(wstring_view value) { - static_assert(std::is_same::value, ""); + static_assert(std::is_same::value, ""); auto&& it = reserve(value.size()); it = std::copy(value.begin(), value.end(), it); } - template - void write(const Char* s, std::size_t size, const format_specs& specs) { - write_padded(specs, str_writer{s, size}); + template + void write(const Ch* s, size_t size, const format_specs& specs) { + auto width = specs.width != 0 + ? count_code_points(basic_string_view(s, size)) + : 0; + out_ = write_padded(out_, specs, size, width, [=](reserve_iterator it) { + return copy_str(s, s + size, it); + }); } - template - void write(basic_string_view s, const format_specs& specs = {}) { - const Char* data = s.data(); - std::size_t size = s.size(); - if (specs.precision >= 0 && to_unsigned(specs.precision) < size) - size = code_point_index(s, to_unsigned(specs.precision)); - write(data, size, specs); - } - - void write_bytes(string_view bytes, const format_specs& specs) { - write_padded(specs, bytes_writer{bytes}); - } - - template - void write_pointer(UIntPtr value, const format_specs* specs) { - int num_digits = count_digits<4>(value); - auto pw = pointer_writer{value, num_digits}; - if (!specs) return pw(reserve(to_unsigned(num_digits) + 2)); - format_specs specs_copy = *specs; - if (specs_copy.align == align::none) specs_copy.align = align::right; - write_padded(specs_copy, pw); - } -}; - -using writer = basic_writer>; - -template struct is_integral : std::is_integral {}; -template <> struct is_integral : std::true_type {}; -template <> struct is_integral : std::true_type {}; - -template -class arg_formatter_base { - public: - using char_type = typename Range::value_type; - using iterator = typename Range::iterator; - using format_specs = basic_format_specs; - - private: - using writer_type = basic_writer; - writer_type writer_; - format_specs* specs_; - - struct char_writer { - char_type value; - - size_t size() const { return 1; } - size_t width() const { return 1; } - - template void operator()(It&& it) const { *it++ = value; } - }; - - void write_char(char_type value) { - if (specs_) - writer_.write_padded(*specs_, char_writer{value}); - else - writer_.write(value); + template + void write(basic_string_view s, const format_specs& specs = {}) { + out_ = detail::write(out_, s, specs); } void write_pointer(const void* p) { - writer_.write_pointer(internal::to_uintptr(p), specs_); - } - - protected: - writer_type& writer() { return writer_; } - FMT_DEPRECATED format_specs* spec() { return specs_; } - format_specs* specs() { return specs_; } - iterator out() { return writer_.out(); } - - void write(bool value) { - string_view sv(value ? "true" : "false"); - specs_ ? writer_.write(sv, *specs_) : writer_.write(sv); - } - - void write(const char_type* value) { - if (!value) { - FMT_THROW(format_error("string pointer is null")); - } else { - auto length = std::char_traits::length(value); - basic_string_view sv(value, length); - specs_ ? writer_.write(sv, *specs_) : writer_.write(sv); - } - } - - public: - arg_formatter_base(Range r, format_specs* s, locale_ref loc) - : writer_(r, loc), specs_(s) {} - - iterator operator()(monostate) { - FMT_ASSERT(false, "invalid argument type"); - return out(); - } - - template ::value)> - iterator operator()(T value) { - if (specs_) - writer_.write_int(value, *specs_); - else - writer_.write(value); - return out(); - } - - iterator operator()(char_type value) { - internal::handle_char_specs( - specs_, char_spec_handler(*this, static_cast(value))); - return out(); - } - - iterator operator()(bool value) { - if (specs_ && specs_->type) return (*this)(value ? 1 : 0); - write(value != 0); - return out(); - } - - template ::value)> - iterator operator()(T value) { - if (const_check(is_supported_floating_point(value))) - writer_.write(value, specs_ ? *specs_ : format_specs()); - else - FMT_ASSERT(false, "unsupported float argument type"); - return out(); + out_ = write_ptr(out_, to_uintptr(p), specs_); } struct char_spec_handler : ErrorHandler { arg_formatter_base& formatter; - char_type value; + Char value; - char_spec_handler(arg_formatter_base& f, char_type val) + char_spec_handler(arg_formatter_base& f, Char val) : formatter(f), value(val) {} void on_int() { - if (formatter.specs_) - formatter.writer_.write_int(value, *formatter.specs_); - else - formatter.writer_.write(value); + // char is only formatted as int if there are specs. + formatter.write_int(static_cast(value), *formatter.specs_); + } + void on_char() { + if (formatter.specs_) + formatter.out_ = write_char(formatter.out_, value, *formatter.specs_); + else + formatter.write(value); } - void on_char() { formatter.write_char(value); } }; - struct cstring_spec_handler : internal::error_handler { + struct cstring_spec_handler : error_handler { arg_formatter_base& formatter; - const char_type* value; + const Char* value; - cstring_spec_handler(arg_formatter_base& f, const char_type* val) + cstring_spec_handler(arg_formatter_base& f, const Char* val) : formatter(f), value(val) {} void on_string() { formatter.write(value); } void on_pointer() { formatter.write_pointer(value); } }; - iterator operator()(const char_type* value) { - if (!specs_) return write(value), out(); - internal::handle_cstring_type_spec(specs_->type, - cstring_spec_handler(*this, value)); - return out(); + protected: + iterator out() { return out_; } + format_specs* specs() { return specs_; } + + void write(bool value) { + if (specs_) + write(string_view(value ? "true" : "false"), *specs_); + else + out_ = detail::write(out_, value); } - iterator operator()(basic_string_view value) { - if (specs_) { - internal::check_string_type_spec(specs_->type, internal::error_handler()); - writer_.write(value, *specs_); + void write(const Char* value) { + if (!value) { + FMT_THROW(format_error("string pointer is null")); } else { - writer_.write(value); + auto length = std::char_traits::length(value); + basic_string_view sv(value, length); + specs_ ? write(sv, *specs_) : write(sv); } - return out(); + } + + public: + arg_formatter_base(OutputIt out, format_specs* s, locale_ref loc) + : out_(out), locale_(loc), specs_(s) {} + + iterator operator()(monostate) { + FMT_ASSERT(false, "invalid argument type"); + return out_; + } + + template ::value)> + FMT_INLINE iterator operator()(T value) { + if (specs_) + write_int(value, *specs_); + else + out_ = detail::write(out_, value); + return out_; + } + + iterator operator()(Char value) { + handle_char_specs(specs_, + char_spec_handler(*this, static_cast(value))); + return out_; + } + + iterator operator()(bool value) { + if (specs_ && specs_->type) return (*this)(value ? 1 : 0); + write(value != 0); + return out_; + } + + template ::value)> + iterator operator()(T value) { + auto specs = specs_ ? *specs_ : format_specs(); + if (const_check(is_supported_floating_point(value))) + out_ = detail::write(out_, value, specs, locale_); + else + FMT_ASSERT(false, "unsupported float argument type"); + return out_; + } + + iterator operator()(const Char* value) { + if (!specs_) return write(value), out_; + handle_cstring_type_spec(specs_->type, cstring_spec_handler(*this, value)); + return out_; + } + + iterator operator()(basic_string_view value) { + if (specs_) { + check_string_type_spec(specs_->type, error_handler()); + write(value, *specs_); + } else { + write(value); + } + return out_; } iterator operator()(const void* value) { - if (specs_) - check_pointer_type_spec(specs_->type, internal::error_handler()); + if (specs_) check_pointer_type_spec(specs_->type, error_handler()); write_pointer(value); - return out(); + return out_; } }; @@ -2109,7 +2131,7 @@ template class specs_setter { template class numeric_specs_checker { public: - FMT_CONSTEXPR numeric_specs_checker(ErrorHandler& eh, internal::type arg_type) + FMT_CONSTEXPR numeric_specs_checker(ErrorHandler& eh, detail::type arg_type) : error_handler_(eh), arg_type_(arg_type) {} FMT_CONSTEXPR void require_numeric_argument() { @@ -2132,18 +2154,24 @@ template class numeric_specs_checker { private: ErrorHandler& error_handler_; - internal::type arg_type_; + detail::type arg_type_; }; // A format specifier handler that checks if specifiers are consistent with the // argument type. template class specs_checker : public Handler { + private: + numeric_specs_checker checker_; + + // Suppress an MSVC warning about using this in initializer list. + FMT_CONSTEXPR Handler& error_handler() { return *this; } + public: - FMT_CONSTEXPR specs_checker(const Handler& handler, internal::type arg_type) - : Handler(handler), checker_(*this, arg_type) {} + FMT_CONSTEXPR specs_checker(const Handler& handler, detail::type arg_type) + : Handler(handler), checker_(error_handler(), arg_type) {} FMT_CONSTEXPR specs_checker(const specs_checker& other) - : Handler(other), checker_(*this, other.arg_type_) {} + : Handler(other), checker_(error_handler(), other.arg_type_) {} FMT_CONSTEXPR void on_align(align_t align) { if (align == align::numeric) checker_.require_numeric_argument(); @@ -2176,9 +2204,6 @@ template class specs_checker : public Handler { } FMT_CONSTEXPR void end_precision() { checker_.check_precision(); } - - private: - numeric_specs_checker checker_; }; template